2024-12-06 16:00:20,907 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-06 16:00:20,922 main DEBUG Took 0.012687 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-06 16:00:20,923 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-06 16:00:20,923 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-06 16:00:20,924 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-06 16:00:20,926 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 16:00:20,935 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-06 16:00:20,951 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 16:00:20,953 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 16:00:20,954 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 16:00:20,954 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 16:00:20,955 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 16:00:20,955 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 16:00:20,956 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 16:00:20,957 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 16:00:20,957 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 16:00:20,958 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 16:00:20,959 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 16:00:20,959 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 16:00:20,960 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 16:00:20,960 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 16:00:20,961 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 16:00:20,961 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 16:00:20,962 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 16:00:20,962 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 16:00:20,963 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 16:00:20,963 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 16:00:20,964 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 16:00:20,964 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 16:00:20,965 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 16:00:20,965 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 16:00:20,966 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 16:00:20,966 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-06 16:00:20,968 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 16:00:20,970 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-06 16:00:20,972 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-06 16:00:20,973 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-06 16:00:20,975 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-06 16:00:20,975 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-06 16:00:20,986 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-06 16:00:20,990 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-06 16:00:20,992 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-06 16:00:20,992 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-06 16:00:20,993 main DEBUG createAppenders(={Console}) 2024-12-06 16:00:20,994 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-06 16:00:20,994 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-06 16:00:20,994 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-06 16:00:20,995 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-06 16:00:20,996 main DEBUG OutputStream closed 2024-12-06 16:00:20,996 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-06 16:00:20,996 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-06 16:00:20,997 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-06 16:00:21,090 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-06 16:00:21,092 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-06 16:00:21,093 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-06 16:00:21,095 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-06 16:00:21,095 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-06 16:00:21,095 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-06 16:00:21,096 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-06 16:00:21,096 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-06 16:00:21,096 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-06 16:00:21,097 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-06 16:00:21,097 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-06 16:00:21,097 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-06 16:00:21,097 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-06 16:00:21,098 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-06 16:00:21,098 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-06 16:00:21,098 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-06 16:00:21,098 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-06 16:00:21,099 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-06 16:00:21,102 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-06 16:00:21,102 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-06 16:00:21,102 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-06 16:00:21,103 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-06T16:00:21,545 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0 2024-12-06 16:00:21,549 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-06 16:00:21,550 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-06T16:00:21,561 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-12-06T16:00:21,586 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-06T16:00:21,607 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/cluster_d00c8b32-e29e-fabf-e232-f9677adb6979, deleteOnExit=true 2024-12-06T16:00:21,607 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-06T16:00:21,609 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/test.cache.data in system properties and HBase conf 2024-12-06T16:00:21,609 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/hadoop.tmp.dir in system properties and HBase conf 2024-12-06T16:00:21,610 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/hadoop.log.dir in system properties and HBase conf 2024-12-06T16:00:21,610 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-06T16:00:21,611 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-06T16:00:21,611 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-06T16:00:21,723 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-06T16:00:21,848 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T16:00:21,853 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-06T16:00:21,853 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-06T16:00:21,854 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-06T16:00:21,855 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-06T16:00:21,855 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-06T16:00:21,856 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-06T16:00:21,857 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-06T16:00:21,857 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-06T16:00:21,858 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-06T16:00:21,858 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/nfs.dump.dir in system properties and HBase conf 2024-12-06T16:00:21,859 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/java.io.tmpdir in system properties and HBase conf 2024-12-06T16:00:21,859 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-06T16:00:21,860 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-06T16:00:21,861 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-06T16:00:23,115 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-06T16:00:23,283 INFO [Time-limited test {}] log.Log(170): Logging initialized @3264ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-06T16:00:23,473 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T16:00:23,599 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-06T16:00:23,642 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-06T16:00:23,642 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-06T16:00:23,643 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-06T16:00:23,670 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T16:00:23,675 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/hadoop.log.dir/,AVAILABLE} 2024-12-06T16:00:23,676 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-06T16:00:24,149 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bbf242f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/java.io.tmpdir/jetty-localhost-33239-hadoop-hdfs-3_4_1-tests_jar-_-any-119011655721561002/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-06T16:00:24,159 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:33239} 2024-12-06T16:00:24,160 INFO [Time-limited test {}] server.Server(415): Started @4142ms 2024-12-06T16:00:24,820 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T16:00:24,828 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-06T16:00:24,829 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-06T16:00:24,830 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-06T16:00:24,830 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-06T16:00:24,831 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b8e8cdd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/hadoop.log.dir/,AVAILABLE} 2024-12-06T16:00:24,832 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ef5f873{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-06T16:00:25,011 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3dfb1097{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/java.io.tmpdir/jetty-localhost-40907-hadoop-hdfs-3_4_1-tests_jar-_-any-9266644337284477943/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T16:00:25,013 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@16142ff{HTTP/1.1, (http/1.1)}{localhost:40907} 2024-12-06T16:00:25,013 INFO [Time-limited test {}] server.Server(415): Started @4995ms 2024-12-06T16:00:25,096 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-06T16:00:25,638 WARN [Thread-70 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/cluster_d00c8b32-e29e-fabf-e232-f9677adb6979/data/data1/current/BP-1739743015-172.17.0.2-1733500822711/current, will proceed with Du for space computation calculation, 2024-12-06T16:00:25,638 WARN [Thread-71 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/cluster_d00c8b32-e29e-fabf-e232-f9677adb6979/data/data2/current/BP-1739743015-172.17.0.2-1733500822711/current, will proceed with Du for space computation calculation, 2024-12-06T16:00:25,712 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-06T16:00:25,777 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x77be8d689a6aa63d with lease ID 0x3ed82f399d390016: Processing first storage report for DS-ca163e4c-4d2b-47d8-a566-a78b046d007d from datanode DatanodeRegistration(127.0.0.1:43971, datanodeUuid=1dbf6204-36ad-4faa-abd2-b24cb5b3dc7a, infoPort=38633, infoSecurePort=0, ipcPort=38727, storageInfo=lv=-57;cid=testClusterID;nsid=1789971697;c=1733500822711) 2024-12-06T16:00:25,779 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x77be8d689a6aa63d with lease ID 0x3ed82f399d390016: from storage DS-ca163e4c-4d2b-47d8-a566-a78b046d007d node DatanodeRegistration(127.0.0.1:43971, datanodeUuid=1dbf6204-36ad-4faa-abd2-b24cb5b3dc7a, infoPort=38633, infoSecurePort=0, ipcPort=38727, storageInfo=lv=-57;cid=testClusterID;nsid=1789971697;c=1733500822711), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-12-06T16:00:25,779 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x77be8d689a6aa63d with lease ID 0x3ed82f399d390016: Processing first storage report for DS-fe646372-2ba7-42df-9706-cac489f9732e from datanode DatanodeRegistration(127.0.0.1:43971, datanodeUuid=1dbf6204-36ad-4faa-abd2-b24cb5b3dc7a, infoPort=38633, infoSecurePort=0, ipcPort=38727, storageInfo=lv=-57;cid=testClusterID;nsid=1789971697;c=1733500822711) 2024-12-06T16:00:25,780 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x77be8d689a6aa63d with lease ID 0x3ed82f399d390016: from storage DS-fe646372-2ba7-42df-9706-cac489f9732e node DatanodeRegistration(127.0.0.1:43971, datanodeUuid=1dbf6204-36ad-4faa-abd2-b24cb5b3dc7a, infoPort=38633, infoSecurePort=0, ipcPort=38727, storageInfo=lv=-57;cid=testClusterID;nsid=1789971697;c=1733500822711), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-06T16:00:25,831 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0 2024-12-06T16:00:25,920 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/cluster_d00c8b32-e29e-fabf-e232-f9677adb6979/zookeeper_0, clientPort=52024, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/cluster_d00c8b32-e29e-fabf-e232-f9677adb6979/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/cluster_d00c8b32-e29e-fabf-e232-f9677adb6979/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-06T16:00:25,938 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52024 2024-12-06T16:00:25,952 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T16:00:25,956 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T16:00:26,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741825_1001 (size=7) 2024-12-06T16:00:26,651 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d with version=8 2024-12-06T16:00:26,651 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/hbase-staging 2024-12-06T16:00:26,785 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-06T16:00:27,094 INFO [Time-limited test {}] client.ConnectionUtils(128): master/85bef17d9292:0 server-side Connection retries=45 2024-12-06T16:00:27,109 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T16:00:27,109 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T16:00:27,116 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T16:00:27,116 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T16:00:27,116 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T16:00:27,285 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-06T16:00:27,376 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-06T16:00:27,390 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-06T16:00:27,395 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T16:00:27,433 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 6090 (auto-detected) 2024-12-06T16:00:27,434 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-06T16:00:27,460 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45907 2024-12-06T16:00:27,491 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45907 connecting to ZooKeeper ensemble=127.0.0.1:52024 2024-12-06T16:00:27,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:459070x0, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T16:00:27,540 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45907-0x100680f2a0d0000 connected 2024-12-06T16:00:27,578 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T16:00:27,581 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T16:00:27,595 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45907-0x100680f2a0d0000, quorum=127.0.0.1:52024, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T16:00:27,600 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d, hbase.cluster.distributed=false 2024-12-06T16:00:27,631 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45907-0x100680f2a0d0000, quorum=127.0.0.1:52024, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T16:00:27,638 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45907 2024-12-06T16:00:27,638 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45907 2024-12-06T16:00:27,639 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45907 2024-12-06T16:00:27,644 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45907 2024-12-06T16:00:27,648 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45907 2024-12-06T16:00:27,771 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/85bef17d9292:0 server-side Connection retries=45 2024-12-06T16:00:27,773 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T16:00:27,774 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T16:00:27,774 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T16:00:27,774 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T16:00:27,774 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T16:00:27,777 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T16:00:27,780 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T16:00:27,781 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46105 2024-12-06T16:00:27,783 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46105 connecting to ZooKeeper ensemble=127.0.0.1:52024 2024-12-06T16:00:27,784 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T16:00:27,789 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T16:00:27,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:461050x0, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T16:00:27,798 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46105-0x100680f2a0d0001 connected 2024-12-06T16:00:27,798 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46105-0x100680f2a0d0001, quorum=127.0.0.1:52024, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T16:00:27,802 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T16:00:27,811 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T16:00:27,814 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46105-0x100680f2a0d0001, quorum=127.0.0.1:52024, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T16:00:27,819 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46105-0x100680f2a0d0001, quorum=127.0.0.1:52024, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T16:00:27,820 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46105 2024-12-06T16:00:27,820 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46105 2024-12-06T16:00:27,820 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46105 2024-12-06T16:00:27,821 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46105 2024-12-06T16:00:27,823 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46105 2024-12-06T16:00:27,840 DEBUG [M:0;85bef17d9292:45907 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;85bef17d9292:45907 2024-12-06T16:00:27,841 INFO [master/85bef17d9292:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/85bef17d9292,45907,1733500826847 2024-12-06T16:00:27,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x100680f2a0d0001, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T16:00:27,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x100680f2a0d0000, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T16:00:27,850 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45907-0x100680f2a0d0000, quorum=127.0.0.1:52024, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/85bef17d9292,45907,1733500826847 2024-12-06T16:00:27,873 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x100680f2a0d0000, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T16:00:27,873 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x100680f2a0d0001, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T16:00:27,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x100680f2a0d0001, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T16:00:27,875 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45907-0x100680f2a0d0000, quorum=127.0.0.1:52024, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T16:00:27,876 INFO [master/85bef17d9292:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/85bef17d9292,45907,1733500826847 from backup master directory 2024-12-06T16:00:27,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x100680f2a0d0000, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/85bef17d9292,45907,1733500826847 2024-12-06T16:00:27,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x100680f2a0d0000, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T16:00:27,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x100680f2a0d0001, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T16:00:27,879 WARN [master/85bef17d9292:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T16:00:27,880 INFO [master/85bef17d9292:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=85bef17d9292,45907,1733500826847 2024-12-06T16:00:27,882 INFO [master/85bef17d9292:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-06T16:00:27,883 INFO [master/85bef17d9292:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-06T16:00:27,941 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/hbase.id] with ID: 9c112d9a-bae3-44dc-8f74-41feaf983fb0 2024-12-06T16:00:27,941 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/.tmp/hbase.id 2024-12-06T16:00:27,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741826_1002 (size=42) 2024-12-06T16:00:28,355 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/.tmp/hbase.id]:[hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/hbase.id] 2024-12-06T16:00:28,405 INFO [master/85bef17d9292:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T16:00:28,409 INFO [master/85bef17d9292:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-06T16:00:28,429 INFO [master/85bef17d9292:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-12-06T16:00:28,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x100680f2a0d0000, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T16:00:28,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x100680f2a0d0001, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T16:00:28,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741827_1003 (size=196) 2024-12-06T16:00:28,866 INFO [master/85bef17d9292:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T16:00:28,868 INFO [master/85bef17d9292:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-06T16:00:28,886 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:28,890 INFO [master/85bef17d9292:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T16:00:28,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741828_1004 (size=1189) 2024-12-06T16:00:29,346 INFO [master/85bef17d9292:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/MasterData/data/master/store 2024-12-06T16:00:29,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741829_1005 (size=34) 2024-12-06T16:00:29,775 INFO [master/85bef17d9292:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-06T16:00:29,778 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T16:00:29,780 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T16:00:29,780 INFO [master/85bef17d9292:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T16:00:29,781 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T16:00:29,783 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T16:00:29,783 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T16:00:29,783 INFO [master/85bef17d9292:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T16:00:29,785 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733500829780Disabling compacts and flushes for region at 1733500829780Disabling writes for close at 1733500829783 (+3 ms)Writing region close event to WAL at 1733500829783Closed at 1733500829783 2024-12-06T16:00:29,787 WARN [master/85bef17d9292:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/MasterData/data/master/store/.initializing 2024-12-06T16:00:29,788 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/MasterData/WALs/85bef17d9292,45907,1733500826847 2024-12-06T16:00:29,798 INFO [master/85bef17d9292:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T16:00:29,813 INFO [master/85bef17d9292:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=85bef17d9292%2C45907%2C1733500826847, suffix=, logDir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/MasterData/WALs/85bef17d9292,45907,1733500826847, archiveDir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/MasterData/oldWALs, maxLogs=10 2024-12-06T16:00:29,845 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/MasterData/WALs/85bef17d9292,45907,1733500826847/85bef17d9292%2C45907%2C1733500826847.1733500829820, exclude list is [], retry=0 2024-12-06T16:00:29,861 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43971,DS-ca163e4c-4d2b-47d8-a566-a78b046d007d,DISK] 2024-12-06T16:00:29,865 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-06T16:00:29,902 INFO [master/85bef17d9292:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/MasterData/WALs/85bef17d9292,45907,1733500826847/85bef17d9292%2C45907%2C1733500826847.1733500829820 2024-12-06T16:00:29,903 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38633:38633)] 2024-12-06T16:00:29,903 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-06T16:00:29,904 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T16:00:29,907 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T16:00:29,908 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T16:00:29,946 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T16:00:29,972 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-06T16:00:29,976 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:00:29,979 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T16:00:29,979 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T16:00:29,982 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-06T16:00:29,983 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:00:29,984 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T16:00:29,984 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T16:00:29,987 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-06T16:00:29,987 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:00:29,988 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T16:00:29,989 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T16:00:29,991 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-06T16:00:29,991 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:00:29,992 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T16:00:29,992 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T16:00:29,996 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-06T16:00:29,997 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-06T16:00:30,002 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T16:00:30,003 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T16:00:30,007 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T16:00:30,012 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T16:00:30,016 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T16:00:30,018 INFO [master/85bef17d9292:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65948796, jitterRate=-0.017286360263824463}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T16:00:30,025 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733500829921Initializing all the Stores at 1733500829923 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733500829924 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733500829925 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733500829925Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733500829925Cleaning up temporary data from old regions at 1733500830003 (+78 ms)Region opened successfully at 1733500830025 (+22 ms) 2024-12-06T16:00:30,026 INFO [master/85bef17d9292:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-06T16:00:30,059 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7847f287, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=85bef17d9292/172.17.0.2:0 2024-12-06T16:00:30,092 INFO [master/85bef17d9292:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-06T16:00:30,103 INFO [master/85bef17d9292:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-06T16:00:30,104 INFO [master/85bef17d9292:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-06T16:00:30,107 INFO [master/85bef17d9292:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-06T16:00:30,108 INFO [master/85bef17d9292:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-06T16:00:30,114 INFO [master/85bef17d9292:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-06T16:00:30,114 INFO [master/85bef17d9292:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-06T16:00:30,140 INFO [master/85bef17d9292:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-06T16:00:30,149 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45907-0x100680f2a0d0000, quorum=127.0.0.1:52024, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-06T16:00:30,151 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-06T16:00:30,153 INFO [master/85bef17d9292:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-06T16:00:30,155 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45907-0x100680f2a0d0000, quorum=127.0.0.1:52024, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-06T16:00:30,156 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-06T16:00:30,158 INFO [master/85bef17d9292:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-06T16:00:30,162 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45907-0x100680f2a0d0000, quorum=127.0.0.1:52024, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-06T16:00:30,163 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-06T16:00:30,165 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45907-0x100680f2a0d0000, quorum=127.0.0.1:52024, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-06T16:00:30,166 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-06T16:00:30,185 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45907-0x100680f2a0d0000, quorum=127.0.0.1:52024, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-06T16:00:30,187 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-06T16:00:30,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x100680f2a0d0000, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T16:00:30,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x100680f2a0d0001, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T16:00:30,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x100680f2a0d0000, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T16:00:30,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x100680f2a0d0001, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T16:00:30,194 INFO [master/85bef17d9292:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=85bef17d9292,45907,1733500826847, sessionid=0x100680f2a0d0000, setting cluster-up flag (Was=false) 2024-12-06T16:00:30,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x100680f2a0d0000, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T16:00:30,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x100680f2a0d0001, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T16:00:30,213 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-06T16:00:30,215 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=85bef17d9292,45907,1733500826847 2024-12-06T16:00:30,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x100680f2a0d0000, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T16:00:30,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x100680f2a0d0001, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T16:00:30,227 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-06T16:00:30,229 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=85bef17d9292,45907,1733500826847 2024-12-06T16:00:30,238 INFO [master/85bef17d9292:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-06T16:00:30,316 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-06T16:00:30,327 INFO [master/85bef17d9292:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-06T16:00:30,328 INFO [RS:0;85bef17d9292:46105 {}] regionserver.HRegionServer(746): ClusterId : 9c112d9a-bae3-44dc-8f74-41feaf983fb0 2024-12-06T16:00:30,331 DEBUG [RS:0;85bef17d9292:46105 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T16:00:30,333 INFO [master/85bef17d9292:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-06T16:00:30,336 DEBUG [RS:0;85bef17d9292:46105 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T16:00:30,336 DEBUG [RS:0;85bef17d9292:46105 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T16:00:30,340 DEBUG [RS:0;85bef17d9292:46105 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T16:00:30,341 DEBUG [RS:0;85bef17d9292:46105 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41bf790c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=85bef17d9292/172.17.0.2:0 2024-12-06T16:00:30,340 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 85bef17d9292,45907,1733500826847 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-06T16:00:30,348 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/85bef17d9292:0, corePoolSize=5, maxPoolSize=5 2024-12-06T16:00:30,348 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/85bef17d9292:0, corePoolSize=5, maxPoolSize=5 2024-12-06T16:00:30,348 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/85bef17d9292:0, corePoolSize=5, maxPoolSize=5 2024-12-06T16:00:30,349 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/85bef17d9292:0, corePoolSize=5, maxPoolSize=5 2024-12-06T16:00:30,349 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/85bef17d9292:0, corePoolSize=10, maxPoolSize=10 2024-12-06T16:00:30,349 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/85bef17d9292:0, corePoolSize=1, maxPoolSize=1 2024-12-06T16:00:30,349 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/85bef17d9292:0, corePoolSize=2, maxPoolSize=2 2024-12-06T16:00:30,350 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/85bef17d9292:0, corePoolSize=1, maxPoolSize=1 2024-12-06T16:00:30,354 INFO [master/85bef17d9292:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733500860354 2024-12-06T16:00:30,356 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T16:00:30,356 INFO [master/85bef17d9292:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-06T16:00:30,356 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-06T16:00:30,357 DEBUG [RS:0;85bef17d9292:46105 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;85bef17d9292:46105 2024-12-06T16:00:30,357 INFO [master/85bef17d9292:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-06T16:00:30,362 INFO [RS:0;85bef17d9292:46105 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T16:00:30,362 INFO [RS:0;85bef17d9292:46105 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T16:00:30,362 DEBUG [RS:0;85bef17d9292:46105 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T16:00:30,362 INFO [master/85bef17d9292:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-06T16:00:30,363 INFO [master/85bef17d9292:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-06T16:00:30,363 INFO [master/85bef17d9292:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-06T16:00:30,363 INFO [master/85bef17d9292:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-06T16:00:30,364 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:00:30,364 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-06T16:00:30,365 INFO [RS:0;85bef17d9292:46105 {}] regionserver.HRegionServer(2659): reportForDuty to master=85bef17d9292,45907,1733500826847 with port=46105, startcode=1733500827732 2024-12-06T16:00:30,364 INFO [master/85bef17d9292:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T16:00:30,368 INFO [master/85bef17d9292:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-06T16:00:30,370 INFO [master/85bef17d9292:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-06T16:00:30,370 INFO [master/85bef17d9292:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-06T16:00:30,373 INFO [master/85bef17d9292:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-06T16:00:30,373 INFO [master/85bef17d9292:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-06T16:00:30,376 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/85bef17d9292:0:becomeActiveMaster-HFileCleaner.large.0-1733500830375,5,FailOnTimeoutGroup] 2024-12-06T16:00:30,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741831_1007 (size=1321) 2024-12-06T16:00:30,379 DEBUG [RS:0;85bef17d9292:46105 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T16:00:30,384 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/85bef17d9292:0:becomeActiveMaster-HFileCleaner.small.0-1733500830376,5,FailOnTimeoutGroup] 2024-12-06T16:00:30,384 INFO [master/85bef17d9292:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T16:00:30,384 INFO [master/85bef17d9292:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-06T16:00:30,386 INFO [master/85bef17d9292:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-06T16:00:30,386 INFO [master/85bef17d9292:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-06T16:00:30,413 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47737, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T16:00:30,421 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45907 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 85bef17d9292,46105,1733500827732 2024-12-06T16:00:30,424 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45907 {}] master.ServerManager(517): Registering regionserver=85bef17d9292,46105,1733500827732 2024-12-06T16:00:30,441 DEBUG [RS:0;85bef17d9292:46105 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d 2024-12-06T16:00:30,442 DEBUG [RS:0;85bef17d9292:46105 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46133 2024-12-06T16:00:30,442 DEBUG [RS:0;85bef17d9292:46105 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T16:00:30,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x100680f2a0d0000, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T16:00:30,448 DEBUG [RS:0;85bef17d9292:46105 {}] zookeeper.ZKUtil(111): regionserver:46105-0x100680f2a0d0001, quorum=127.0.0.1:52024, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/85bef17d9292,46105,1733500827732 2024-12-06T16:00:30,448 WARN [RS:0;85bef17d9292:46105 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T16:00:30,448 INFO [RS:0;85bef17d9292:46105 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T16:00:30,448 DEBUG [RS:0;85bef17d9292:46105 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/WALs/85bef17d9292,46105,1733500827732 2024-12-06T16:00:30,450 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [85bef17d9292,46105,1733500827732] 2024-12-06T16:00:30,475 INFO [RS:0;85bef17d9292:46105 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T16:00:30,489 INFO [RS:0;85bef17d9292:46105 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T16:00:30,494 INFO [RS:0;85bef17d9292:46105 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T16:00:30,494 INFO [RS:0;85bef17d9292:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T16:00:30,495 INFO [RS:0;85bef17d9292:46105 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T16:00:30,501 INFO [RS:0;85bef17d9292:46105 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T16:00:30,502 INFO [RS:0;85bef17d9292:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T16:00:30,502 DEBUG [RS:0;85bef17d9292:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/85bef17d9292:0, corePoolSize=1, maxPoolSize=1 2024-12-06T16:00:30,503 DEBUG [RS:0;85bef17d9292:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/85bef17d9292:0, corePoolSize=1, maxPoolSize=1 2024-12-06T16:00:30,503 DEBUG [RS:0;85bef17d9292:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/85bef17d9292:0, corePoolSize=1, maxPoolSize=1 2024-12-06T16:00:30,503 DEBUG [RS:0;85bef17d9292:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/85bef17d9292:0, corePoolSize=1, maxPoolSize=1 2024-12-06T16:00:30,503 DEBUG [RS:0;85bef17d9292:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/85bef17d9292:0, corePoolSize=1, maxPoolSize=1 2024-12-06T16:00:30,503 DEBUG [RS:0;85bef17d9292:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/85bef17d9292:0, corePoolSize=2, maxPoolSize=2 2024-12-06T16:00:30,503 DEBUG [RS:0;85bef17d9292:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0, corePoolSize=1, maxPoolSize=1 2024-12-06T16:00:30,503 DEBUG [RS:0;85bef17d9292:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/85bef17d9292:0, corePoolSize=1, maxPoolSize=1 2024-12-06T16:00:30,504 DEBUG [RS:0;85bef17d9292:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/85bef17d9292:0, corePoolSize=1, maxPoolSize=1 2024-12-06T16:00:30,504 DEBUG [RS:0;85bef17d9292:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/85bef17d9292:0, corePoolSize=1, maxPoolSize=1 2024-12-06T16:00:30,504 DEBUG [RS:0;85bef17d9292:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/85bef17d9292:0, corePoolSize=1, maxPoolSize=1 2024-12-06T16:00:30,504 DEBUG [RS:0;85bef17d9292:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/85bef17d9292:0, corePoolSize=1, maxPoolSize=1 2024-12-06T16:00:30,504 DEBUG [RS:0;85bef17d9292:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/85bef17d9292:0, corePoolSize=3, maxPoolSize=3 2024-12-06T16:00:30,505 DEBUG [RS:0;85bef17d9292:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0, corePoolSize=3, maxPoolSize=3 2024-12-06T16:00:30,505 INFO [RS:0;85bef17d9292:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T16:00:30,506 INFO [RS:0;85bef17d9292:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T16:00:30,506 INFO [RS:0;85bef17d9292:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T16:00:30,506 INFO [RS:0;85bef17d9292:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T16:00:30,506 INFO [RS:0;85bef17d9292:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T16:00:30,507 INFO [RS:0;85bef17d9292:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=85bef17d9292,46105,1733500827732-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T16:00:30,527 INFO [RS:0;85bef17d9292:46105 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T16:00:30,529 INFO [RS:0;85bef17d9292:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=85bef17d9292,46105,1733500827732-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T16:00:30,530 INFO [RS:0;85bef17d9292:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T16:00:30,530 INFO [RS:0;85bef17d9292:46105 {}] regionserver.Replication(171): 85bef17d9292,46105,1733500827732 started 2024-12-06T16:00:30,549 INFO [RS:0;85bef17d9292:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T16:00:30,550 INFO [RS:0;85bef17d9292:46105 {}] regionserver.HRegionServer(1482): Serving as 85bef17d9292,46105,1733500827732, RpcServer on 85bef17d9292/172.17.0.2:46105, sessionid=0x100680f2a0d0001 2024-12-06T16:00:30,550 DEBUG [RS:0;85bef17d9292:46105 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T16:00:30,550 DEBUG [RS:0;85bef17d9292:46105 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 85bef17d9292,46105,1733500827732 2024-12-06T16:00:30,551 DEBUG [RS:0;85bef17d9292:46105 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '85bef17d9292,46105,1733500827732' 2024-12-06T16:00:30,551 DEBUG [RS:0;85bef17d9292:46105 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T16:00:30,552 DEBUG [RS:0;85bef17d9292:46105 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T16:00:30,553 DEBUG [RS:0;85bef17d9292:46105 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T16:00:30,553 DEBUG [RS:0;85bef17d9292:46105 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T16:00:30,553 DEBUG [RS:0;85bef17d9292:46105 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 85bef17d9292,46105,1733500827732 2024-12-06T16:00:30,553 DEBUG [RS:0;85bef17d9292:46105 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '85bef17d9292,46105,1733500827732' 2024-12-06T16:00:30,553 DEBUG [RS:0;85bef17d9292:46105 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T16:00:30,554 DEBUG [RS:0;85bef17d9292:46105 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T16:00:30,554 DEBUG [RS:0;85bef17d9292:46105 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T16:00:30,554 INFO [RS:0;85bef17d9292:46105 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T16:00:30,554 INFO [RS:0;85bef17d9292:46105 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T16:00:30,660 INFO [RS:0;85bef17d9292:46105 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T16:00:30,664 INFO [RS:0;85bef17d9292:46105 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=85bef17d9292%2C46105%2C1733500827732, suffix=, logDir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/WALs/85bef17d9292,46105,1733500827732, archiveDir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/oldWALs, maxLogs=32 2024-12-06T16:00:30,681 DEBUG [RS:0;85bef17d9292:46105 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/WALs/85bef17d9292,46105,1733500827732/85bef17d9292%2C46105%2C1733500827732.1733500830666, exclude list is [], retry=0 2024-12-06T16:00:30,686 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43971,DS-ca163e4c-4d2b-47d8-a566-a78b046d007d,DISK] 2024-12-06T16:00:30,689 INFO [RS:0;85bef17d9292:46105 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/WALs/85bef17d9292,46105,1733500827732/85bef17d9292%2C46105%2C1733500827732.1733500830666 2024-12-06T16:00:30,690 DEBUG [RS:0;85bef17d9292:46105 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38633:38633)] 2024-12-06T16:00:30,781 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-06T16:00:30,782 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d 2024-12-06T16:00:30,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741833_1009 (size=32) 2024-12-06T16:00:31,193 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T16:00:31,196 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-06T16:00:31,199 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T16:00:31,199 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:00:31,200 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T16:00:31,201 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-06T16:00:31,204 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T16:00:31,204 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:00:31,205 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T16:00:31,205 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-06T16:00:31,208 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T16:00:31,208 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:00:31,209 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T16:00:31,209 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-06T16:00:31,212 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T16:00:31,212 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:00:31,213 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T16:00:31,213 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T16:00:31,215 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/hbase/meta/1588230740 2024-12-06T16:00:31,216 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/hbase/meta/1588230740 2024-12-06T16:00:31,219 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T16:00:31,219 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T16:00:31,220 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T16:00:31,223 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T16:00:31,227 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T16:00:31,228 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75078364, jitterRate=0.1187548041343689}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T16:00:31,230 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733500831194Initializing all the Stores at 1733500831196 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733500831196Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733500831196Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733500831196Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733500831196Cleaning up temporary data from old regions at 1733500831219 (+23 ms)Region opened successfully at 1733500831230 (+11 ms) 2024-12-06T16:00:31,230 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T16:00:31,230 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T16:00:31,230 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T16:00:31,231 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T16:00:31,231 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T16:00:31,232 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T16:00:31,232 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733500831230Disabling compacts and flushes for region at 1733500831230Disabling writes for close at 1733500831231 (+1 ms)Writing region close event to WAL at 1733500831231Closed at 1733500831232 (+1 ms) 2024-12-06T16:00:31,235 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T16:00:31,235 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-06T16:00:31,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-06T16:00:31,250 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T16:00:31,253 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-06T16:00:31,406 DEBUG [85bef17d9292:45907 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-06T16:00:31,419 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=85bef17d9292,46105,1733500827732 2024-12-06T16:00:31,426 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 85bef17d9292,46105,1733500827732, state=OPENING 2024-12-06T16:00:31,432 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-06T16:00:31,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x100680f2a0d0000, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T16:00:31,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x100680f2a0d0001, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T16:00:31,435 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T16:00:31,435 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T16:00:31,436 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T16:00:31,438 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=85bef17d9292,46105,1733500827732}] 2024-12-06T16:00:31,614 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T16:00:31,617 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38731, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T16:00:31,628 INFO [RS_OPEN_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-06T16:00:31,629 INFO [RS_OPEN_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T16:00:31,629 INFO [RS_OPEN_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-06T16:00:31,632 INFO [RS_OPEN_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=85bef17d9292%2C46105%2C1733500827732.meta, suffix=.meta, logDir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/WALs/85bef17d9292,46105,1733500827732, archiveDir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/oldWALs, maxLogs=32 2024-12-06T16:00:31,650 DEBUG [RS_OPEN_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/WALs/85bef17d9292,46105,1733500827732/85bef17d9292%2C46105%2C1733500827732.meta.1733500831634.meta, exclude list is [], retry=0 2024-12-06T16:00:31,654 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43971,DS-ca163e4c-4d2b-47d8-a566-a78b046d007d,DISK] 2024-12-06T16:00:31,658 INFO [RS_OPEN_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/WALs/85bef17d9292,46105,1733500827732/85bef17d9292%2C46105%2C1733500827732.meta.1733500831634.meta 2024-12-06T16:00:31,659 DEBUG [RS_OPEN_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38633:38633)] 2024-12-06T16:00:31,659 DEBUG [RS_OPEN_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-06T16:00:31,661 DEBUG [RS_OPEN_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-06T16:00:31,664 DEBUG [RS_OPEN_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-06T16:00:31,669 INFO [RS_OPEN_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-06T16:00:31,673 DEBUG [RS_OPEN_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-06T16:00:31,674 DEBUG [RS_OPEN_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T16:00:31,674 DEBUG [RS_OPEN_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-06T16:00:31,674 DEBUG [RS_OPEN_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-06T16:00:31,677 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-06T16:00:31,679 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T16:00:31,679 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:00:31,680 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T16:00:31,680 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-06T16:00:31,682 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T16:00:31,682 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:00:31,683 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T16:00:31,683 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-06T16:00:31,684 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T16:00:31,684 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:00:31,685 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T16:00:31,685 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-06T16:00:31,687 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T16:00:31,687 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:00:31,687 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T16:00:31,688 DEBUG [RS_OPEN_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T16:00:31,689 DEBUG [RS_OPEN_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/hbase/meta/1588230740 2024-12-06T16:00:31,691 DEBUG [RS_OPEN_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/hbase/meta/1588230740 2024-12-06T16:00:31,693 DEBUG [RS_OPEN_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T16:00:31,694 DEBUG [RS_OPEN_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T16:00:31,695 DEBUG [RS_OPEN_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T16:00:31,697 DEBUG [RS_OPEN_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T16:00:31,698 INFO [RS_OPEN_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60498975, jitterRate=-0.09849502146244049}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T16:00:31,699 DEBUG [RS_OPEN_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-06T16:00:31,700 DEBUG [RS_OPEN_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733500831675Writing region info on filesystem at 1733500831675Initializing all the Stores at 1733500831677 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733500831677Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733500831677Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733500831677Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733500831677Cleaning up temporary data from old regions at 1733500831694 (+17 ms)Running coprocessor post-open hooks at 1733500831699 (+5 ms)Region opened successfully at 1733500831700 (+1 ms) 2024-12-06T16:00:31,708 INFO [RS_OPEN_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733500831604 2024-12-06T16:00:31,721 DEBUG [RS_OPEN_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-06T16:00:31,721 INFO [RS_OPEN_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-06T16:00:31,724 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=85bef17d9292,46105,1733500827732 2024-12-06T16:00:31,726 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 85bef17d9292,46105,1733500827732, state=OPEN 2024-12-06T16:00:31,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x100680f2a0d0001, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T16:00:31,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x100680f2a0d0000, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T16:00:31,734 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T16:00:31,734 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T16:00:31,734 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:31,740 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-06T16:00:31,741 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=85bef17d9292,46105,1733500827732 in 296 msec 2024-12-06T16:00:31,748 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-06T16:00:31,748 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 501 msec 2024-12-06T16:00:31,750 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T16:00:31,750 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-06T16:00:31,777 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:00:31,779 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:00:31,807 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:00:31,810 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58235, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:00:31,832 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5600 sec 2024-12-06T16:00:31,833 INFO [master/85bef17d9292:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733500831832, completionTime=-1 2024-12-06T16:00:31,861 INFO [master/85bef17d9292:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-06T16:00:31,861 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-06T16:00:31,898 INFO [master/85bef17d9292:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-06T16:00:31,898 INFO [master/85bef17d9292:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733500891898 2024-12-06T16:00:31,898 INFO [master/85bef17d9292:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733500951898 2024-12-06T16:00:31,898 INFO [master/85bef17d9292:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 36 msec 2024-12-06T16:00:31,901 INFO [master/85bef17d9292:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=85bef17d9292,45907,1733500826847-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T16:00:31,902 INFO [master/85bef17d9292:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=85bef17d9292,45907,1733500826847-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T16:00:31,902 INFO [master/85bef17d9292:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=85bef17d9292,45907,1733500826847-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T16:00:31,904 INFO [master/85bef17d9292:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-85bef17d9292:45907, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T16:00:31,904 INFO [master/85bef17d9292:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-06T16:00:31,905 INFO [master/85bef17d9292:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-06T16:00:31,910 DEBUG [master/85bef17d9292:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-06T16:00:31,935 INFO [master/85bef17d9292:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.055sec 2024-12-06T16:00:31,936 INFO [master/85bef17d9292:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-06T16:00:31,937 INFO [master/85bef17d9292:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-06T16:00:31,938 INFO [master/85bef17d9292:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-06T16:00:31,939 INFO [master/85bef17d9292:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-06T16:00:31,939 INFO [master/85bef17d9292:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-06T16:00:31,940 INFO [master/85bef17d9292:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=85bef17d9292,45907,1733500826847-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T16:00:31,940 INFO [master/85bef17d9292:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=85bef17d9292,45907,1733500826847-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-06T16:00:31,948 DEBUG [master/85bef17d9292:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-06T16:00:31,949 INFO [master/85bef17d9292:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-06T16:00:31,950 INFO [master/85bef17d9292:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=85bef17d9292,45907,1733500826847-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T16:00:32,040 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bed8dc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:32,042 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-06T16:00:32,042 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-06T16:00:32,045 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:00:32,048 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:00:32,055 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:00:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:00:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:00:32,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c86253f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:32,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:00:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:00:32,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:00:32,065 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56584, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:00:32,068 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@738d23fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:32,068 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:00:32,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:00:32,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:00:32,078 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34484, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:00:32,097 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=85bef17d9292,45907,1733500826847 2024-12-06T16:00:32,109 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=400, ProcessCount=11, AvailableMemoryMB=9570 2024-12-06T16:00:32,117 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T16:00:32,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 85bef17d9292,45907,1733500826847 2024-12-06T16:00:32,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@571ad612 2024-12-06T16:00:32,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T16:00:32,129 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T16:00:32,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-06T16:00:32,135 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T16:00:32,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-06T16:00:32,146 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T16:00:32,148 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-06T16:00:32,148 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:00:32,150 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T16:00:32,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T16:00:32,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741835_1011 (size=959) 2024-12-06T16:00:32,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T16:00:32,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T16:00:32,566 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d 2024-12-06T16:00:32,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741836_1012 (size=53) 2024-12-06T16:00:32,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T16:00:32,977 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T16:00:32,977 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 81bfe42898e92490dcd2dea3186de72d, disabling compactions & flushes 2024-12-06T16:00:32,977 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:32,977 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:32,977 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. after waiting 0 ms 2024-12-06T16:00:32,977 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:32,978 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:32,978 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 81bfe42898e92490dcd2dea3186de72d: Waiting for close lock at 1733500832977Disabling compacts and flushes for region at 1733500832977Disabling writes for close at 1733500832977Writing region close event to WAL at 1733500832977Closed at 1733500832977 2024-12-06T16:00:32,980 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T16:00:32,985 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733500832980"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733500832980"}]},"ts":"1733500832980"} 2024-12-06T16:00:32,991 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T16:00:32,993 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T16:00:32,995 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733500832993"}]},"ts":"1733500832993"} 2024-12-06T16:00:33,000 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-06T16:00:33,003 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81bfe42898e92490dcd2dea3186de72d, ASSIGN}] 2024-12-06T16:00:33,006 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81bfe42898e92490dcd2dea3186de72d, ASSIGN 2024-12-06T16:00:33,008 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=81bfe42898e92490dcd2dea3186de72d, ASSIGN; state=OFFLINE, location=85bef17d9292,46105,1733500827732; forceNewPlan=false, retain=false 2024-12-06T16:00:33,160 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=81bfe42898e92490dcd2dea3186de72d, regionState=OPENING, regionLocation=85bef17d9292,46105,1733500827732 2024-12-06T16:00:33,164 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=81bfe42898e92490dcd2dea3186de72d, ASSIGN because future has completed 2024-12-06T16:00:33,165 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732}] 2024-12-06T16:00:33,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T16:00:33,326 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:33,326 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} 2024-12-06T16:00:33,326 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:33,327 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T16:00:33,327 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:33,327 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:33,329 INFO [StoreOpener-81bfe42898e92490dcd2dea3186de72d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:33,332 INFO [StoreOpener-81bfe42898e92490dcd2dea3186de72d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81bfe42898e92490dcd2dea3186de72d columnFamilyName A 2024-12-06T16:00:33,332 DEBUG [StoreOpener-81bfe42898e92490dcd2dea3186de72d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:00:33,333 INFO [StoreOpener-81bfe42898e92490dcd2dea3186de72d-1 {}] regionserver.HStore(327): Store=81bfe42898e92490dcd2dea3186de72d/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T16:00:33,333 INFO [StoreOpener-81bfe42898e92490dcd2dea3186de72d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:33,336 INFO [StoreOpener-81bfe42898e92490dcd2dea3186de72d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81bfe42898e92490dcd2dea3186de72d columnFamilyName B 2024-12-06T16:00:33,336 DEBUG [StoreOpener-81bfe42898e92490dcd2dea3186de72d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:00:33,337 INFO [StoreOpener-81bfe42898e92490dcd2dea3186de72d-1 {}] regionserver.HStore(327): Store=81bfe42898e92490dcd2dea3186de72d/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T16:00:33,337 INFO [StoreOpener-81bfe42898e92490dcd2dea3186de72d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:33,340 INFO [StoreOpener-81bfe42898e92490dcd2dea3186de72d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81bfe42898e92490dcd2dea3186de72d columnFamilyName C 2024-12-06T16:00:33,340 DEBUG [StoreOpener-81bfe42898e92490dcd2dea3186de72d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:00:33,341 INFO [StoreOpener-81bfe42898e92490dcd2dea3186de72d-1 {}] regionserver.HStore(327): Store=81bfe42898e92490dcd2dea3186de72d/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T16:00:33,341 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:33,342 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:33,343 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:33,345 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:33,345 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:33,346 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T16:00:33,348 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:33,352 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T16:00:33,353 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 81bfe42898e92490dcd2dea3186de72d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69322924, jitterRate=0.03299206495285034}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T16:00:33,353 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:33,355 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 81bfe42898e92490dcd2dea3186de72d: Running coprocessor pre-open hook at 1733500833327Writing region info on filesystem at 1733500833327Initializing all the Stores at 1733500833329 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733500833329Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733500833329Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733500833329Cleaning up temporary data from old regions at 1733500833345 (+16 ms)Running coprocessor post-open hooks at 1733500833353 (+8 ms)Region opened successfully at 1733500833354 (+1 ms) 2024-12-06T16:00:33,357 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., pid=6, masterSystemTime=1733500833319 2024-12-06T16:00:33,361 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:33,361 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:33,362 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=81bfe42898e92490dcd2dea3186de72d, regionState=OPEN, openSeqNum=2, regionLocation=85bef17d9292,46105,1733500827732 2024-12-06T16:00:33,366 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 because future has completed 2024-12-06T16:00:33,372 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-06T16:00:33,372 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 in 203 msec 2024-12-06T16:00:33,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-06T16:00:33,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81bfe42898e92490dcd2dea3186de72d, ASSIGN in 369 msec 2024-12-06T16:00:33,377 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T16:00:33,377 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733500833377"}]},"ts":"1733500833377"} 2024-12-06T16:00:33,381 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-06T16:00:33,382 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T16:00:33,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2450 sec 2024-12-06T16:00:34,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T16:00:34,303 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:00:34,310 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ffdeb1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:34,310 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:00:34,310 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:00:34,311 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:00:34,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:00:34,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:00:34,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d7b1500, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:34,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:00:34,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:00:34,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:00:34,315 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:00:34,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c0d8197, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:34,317 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:00:34,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:00:34,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:00:34,321 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:00:34,324 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T16:00:34,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 85bef17d9292,45907,1733500826847 2024-12-06T16:00:34,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@69abe2c9 2024-12-06T16:00:34,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T16:00:34,330 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52014, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T16:00:34,335 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44ce0b30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:34,335 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:00:34,335 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:00:34,337 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:00:34,337 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:00:34,337 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:00:34,338 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@455d50bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:34,338 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:00:34,338 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:00:34,339 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:00:34,340 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:00:34,342 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@760c0989, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:34,343 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15170e84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:34,343 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:00:34,343 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:00:34,344 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:00:34,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:00:34,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:00:34,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b3d2fd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:34,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:00:34,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:00:34,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:00:34,348 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52038, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:00:34,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c860641, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:34,350 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b4d8404, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:34,350 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:00:34,350 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:00:34,351 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:00:34,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:00:34,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:00:34,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cdb8e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:34,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:00:34,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:00:34,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:00:34,357 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:00:34,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69b21050, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:34,359 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@382062a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:34,359 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:00:34,359 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:00:34,360 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:00:34,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:00:34,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:00:34,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b084ce6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:34,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:00:34,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:00:34,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:00:34,364 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:00:34,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2815dd8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:34,366 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64f5332d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:34,366 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:00:34,366 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:00:34,367 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:00:34,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:00:34,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:00:34,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d96214, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:34,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:00:34,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:00:34,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:00:34,370 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52098, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:00:34,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47cbee8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:34,372 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75ef992a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:34,372 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:00:34,372 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:00:34,373 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:00:34,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:00:34,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:00:34,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d6e4493, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:34,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:00:34,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:00:34,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:00:34,376 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52118, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:00:34,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1455c669, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:34,379 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19215e1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:34,379 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:00:34,379 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:00:34,380 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:00:34,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:00:34,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:00:34,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af79170, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:34,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:00:34,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:00:34,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:00:34,382 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52134, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:00:34,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d11e8ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:34,384 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64696a75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:34,384 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:00:34,384 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:00:34,385 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:00:34,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:00:34,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:00:34,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d774ab5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:34,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:00:34,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:00:34,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:00:34,387 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52154, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:00:34,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@739b8c1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:34,389 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ae6950f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:34,389 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:00:34,389 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:00:34,390 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:00:34,390 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:00:34,391 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:00:34,391 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@355546b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:34,391 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:00:34,391 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:00:34,391 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:00:34,392 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52162, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:00:34,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23600c61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:00:34,399 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:00:34,399 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:00:34,400 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:00:34,401 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:00:34,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:00:34,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:00:34,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:00:34,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:00:34,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:00:34,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:00:34,403 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:00:34,404 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40704, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:00:34,404 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40692, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:00:34,405 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40720, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:00:34,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:00:34,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:00:34,407 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40726, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:00:34,410 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:00:34,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:00:34,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:00:34,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-06T16:00:34,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2] 2024-12-06T16:00:34,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2] 2024-12-06T16:00:34,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2] 2024-12-06T16:00:34,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2] 2024-12-06T16:00:34,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-06T16:00:34,416 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40738, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:00:34,419 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:00:34,423 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:00:34,425 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:00:34,425 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:00:34,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:00:34,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:00:34,427 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:00:34,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:00:34,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:00:34,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:00:34,429 DEBUG [Thread-143 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:00:34,429 DEBUG [Thread-143 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:00:34,432 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40752, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:00:34,432 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40760, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:00:34,432 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:00:34,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2] 2024-12-06T16:00:34,434 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:00:34,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2] 2024-12-06T16:00:34,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2] 2024-12-06T16:00:34,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2] 2024-12-06T16:00:34,448 DEBUG [Thread-154 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:00:34,448 DEBUG [Thread-154 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:00:34,450 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40778, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:00:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2] 2024-12-06T16:00:34,522 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:00:34,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:34,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-06T16:00:34,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81bfe42898e92490dcd2dea3186de72d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-06T16:00:34,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-06T16:00:34,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:34,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:34,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:34,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:34,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:34,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-06T16:00:34,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/9c86b711d19445ea90d0dac966b7547d is 50, key is test_row_0/A:col10/1733500834525/Put/seqid=0 2024-12-06T16:00:34,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741837_1013 (size=14341) 2024-12-06T16:00:34,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-06T16:00:34,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500894748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:34,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:34,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:34,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:34,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:34,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:34,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:34,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500894789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:34,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500894818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:34,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500894824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:34,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500894819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:34,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:34,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:34,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:34,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:34,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:34,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:34,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:34,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:34,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:34,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:34,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:34,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:34,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:34,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:34,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:34,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500894973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:34,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:34,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:34,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:34,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500894974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:34,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500894974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:34,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:34,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:34,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:34,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-06T16:00:34,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:34,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500894975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:34,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:34,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:34,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:34,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500894975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:34,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:34,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:34,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:34,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:34,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:34,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:34,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:34,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:34,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:34,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:35,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-06T16:00:35,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-06T16:00:35,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:35,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:35,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:35,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:35,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:35,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:35,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/9c86b711d19445ea90d0dac966b7547d 2024-12-06T16:00:35,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:35,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500895183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:35,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:35,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:35,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500895192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:35,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:35,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500895193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:35,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:35,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:35,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500895193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:35,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:35,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:35,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:35,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:35,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:35,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:35,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500895194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:35,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:35,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:35,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:35,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-06T16:00:35,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:35,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:35,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:35,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:35,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:35,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:35,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/d3b71cfa7747430e93c2dfce60fa05ee is 50, key is test_row_0/B:col10/1733500834525/Put/seqid=0 2024-12-06T16:00:35,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741838_1014 (size=12001) 2024-12-06T16:00:35,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/d3b71cfa7747430e93c2dfce60fa05ee 2024-12-06T16:00:35,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/780815a6351a4b848606dcdc5d037987 is 50, key is test_row_0/C:col10/1733500834525/Put/seqid=0 2024-12-06T16:00:35,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-06T16:00:35,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:35,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:35,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:35,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:35,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:35,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:35,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741839_1015 (size=12001) 2024-12-06T16:00:35,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:35,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500895502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:35,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:35,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:35,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:35,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:35,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500895502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:35,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:35,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:35,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:35,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:35,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500895514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:35,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:35,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:35,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:35,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:35,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500895514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:35,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:35,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:35,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:35,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:35,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500895514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:35,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:35,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:35,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:35,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-06T16:00:35,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-06T16:00:35,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:35,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:35,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:35,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:35,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:35,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:35,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-06T16:00:35,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:35,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:35,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:35,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:35,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:35,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/780815a6351a4b848606dcdc5d037987 2024-12-06T16:00:35,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/9c86b711d19445ea90d0dac966b7547d as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/9c86b711d19445ea90d0dac966b7547d 2024-12-06T16:00:35,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/9c86b711d19445ea90d0dac966b7547d, entries=200, sequenceid=14, filesize=14.0 K 2024-12-06T16:00:35,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-06T16:00:35,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:35,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:35,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:35,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:35,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:35,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/d3b71cfa7747430e93c2dfce60fa05ee as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/d3b71cfa7747430e93c2dfce60fa05ee 2024-12-06T16:00:35,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/d3b71cfa7747430e93c2dfce60fa05ee, entries=150, sequenceid=14, filesize=11.7 K 2024-12-06T16:00:35,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/780815a6351a4b848606dcdc5d037987 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/780815a6351a4b848606dcdc5d037987 2024-12-06T16:00:35,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/780815a6351a4b848606dcdc5d037987, entries=150, sequenceid=14, filesize=11.7 K 2024-12-06T16:00:35,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=147.60 KB/151140 for 81bfe42898e92490dcd2dea3186de72d in 1448ms, sequenceid=14, compaction requested=false 2024-12-06T16:00:35,979 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-06T16:00:35,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:36,020 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:00:36,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81bfe42898e92490dcd2dea3186de72d 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-12-06T16:00:36,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:36,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/2ad7b5dd0a7e4bdca8ff4cfe3a99ce73 is 50, key is test_row_0/A:col10/1733500834774/Put/seqid=0 2024-12-06T16:00:36,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:36,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500896038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:36,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:36,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500896045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:36,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:36,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500896042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:36,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:36,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:36,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500896061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:36,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:36,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:36,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500896063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:36,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:36,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:36,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:36,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741840_1016 (size=16681) 2024-12-06T16:00:36,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/2ad7b5dd0a7e4bdca8ff4cfe3a99ce73 2024-12-06T16:00:36,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-06T16:00:36,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:36,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:36,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:36,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:36,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:36,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/20b7171476974be3afffa1704fa2efe8 is 50, key is test_row_0/B:col10/1733500834774/Put/seqid=0 2024-12-06T16:00:36,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741841_1017 (size=12001) 2024-12-06T16:00:36,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/20b7171476974be3afffa1704fa2efe8 2024-12-06T16:00:36,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/2b86102c03074554825a233e07750076 is 50, key is test_row_0/C:col10/1733500834774/Put/seqid=0 2024-12-06T16:00:36,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:36,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500896173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:36,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:36,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:36,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500896173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:36,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:36,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:36,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500896174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:36,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:36,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:36,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500896182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:36,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:36,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:36,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500896183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:36,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:36,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741842_1018 (size=12001) 2024-12-06T16:00:36,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/2b86102c03074554825a233e07750076 2024-12-06T16:00:36,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/2ad7b5dd0a7e4bdca8ff4cfe3a99ce73 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/2ad7b5dd0a7e4bdca8ff4cfe3a99ce73 2024-12-06T16:00:36,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-06T16:00:36,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:36,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/2ad7b5dd0a7e4bdca8ff4cfe3a99ce73, entries=250, sequenceid=41, filesize=16.3 K 2024-12-06T16:00:36,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:36,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:36,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:36,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:36,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/20b7171476974be3afffa1704fa2efe8 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/20b7171476974be3afffa1704fa2efe8 2024-12-06T16:00:36,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:36,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/20b7171476974be3afffa1704fa2efe8, entries=150, sequenceid=41, filesize=11.7 K 2024-12-06T16:00:36,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/2b86102c03074554825a233e07750076 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/2b86102c03074554825a233e07750076 2024-12-06T16:00:36,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/2b86102c03074554825a233e07750076, entries=150, sequenceid=41, filesize=11.7 K 2024-12-06T16:00:36,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 81bfe42898e92490dcd2dea3186de72d in 262ms, sequenceid=41, compaction requested=false 2024-12-06T16:00:36,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:36,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-06T16:00:36,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:36,400 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 81bfe42898e92490dcd2dea3186de72d 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-12-06T16:00:36,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/c1437c485fa648a6a170ea8ad91d4d0b is 50, key is test_row_0/A:col10/1733500836054/Put/seqid=0 2024-12-06T16:00:36,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:36,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:36,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741843_1019 (size=12001) 2024-12-06T16:00:36,442 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/c1437c485fa648a6a170ea8ad91d4d0b 2024-12-06T16:00:36,469 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-06T16:00:36,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/06dbeaedd7054946a34244e746431f4f is 50, key is test_row_0/B:col10/1733500836054/Put/seqid=0 2024-12-06T16:00:36,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741844_1020 (size=12001) 2024-12-06T16:00:36,514 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/06dbeaedd7054946a34244e746431f4f 2024-12-06T16:00:36,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:36,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500896528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:36,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/18c416650918464da4bba863a49fcee3 is 50, key is test_row_0/C:col10/1733500836054/Put/seqid=0 2024-12-06T16:00:36,543 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,543 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,543 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:36,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:36,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500896533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:36,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:36,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:36,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500896535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:36,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:36,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:36,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500896540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:36,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:36,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:36,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500896541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:36,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:36,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741845_1021 (size=12001) 2024-12-06T16:00:36,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-06T16:00:36,564 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/18c416650918464da4bba863a49fcee3 2024-12-06T16:00:36,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/c1437c485fa648a6a170ea8ad91d4d0b as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/c1437c485fa648a6a170ea8ad91d4d0b 2024-12-06T16:00:36,599 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/c1437c485fa648a6a170ea8ad91d4d0b, entries=150, sequenceid=50, filesize=11.7 K 2024-12-06T16:00:36,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/06dbeaedd7054946a34244e746431f4f as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/06dbeaedd7054946a34244e746431f4f 2024-12-06T16:00:36,616 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/06dbeaedd7054946a34244e746431f4f, entries=150, sequenceid=50, filesize=11.7 K 2024-12-06T16:00:36,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/18c416650918464da4bba863a49fcee3 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/18c416650918464da4bba863a49fcee3 2024-12-06T16:00:36,637 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/18c416650918464da4bba863a49fcee3, entries=150, sequenceid=50, filesize=11.7 K 2024-12-06T16:00:36,641 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=167.72 KB/171750 for 81bfe42898e92490dcd2dea3186de72d in 241ms, sequenceid=50, compaction requested=true 2024-12-06T16:00:36,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:36,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:36,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-06T16:00:36,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-06T16:00:36,648 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-06T16:00:36,648 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2170 sec 2024-12-06T16:00:36,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.2460 sec 2024-12-06T16:00:36,659 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:00:36,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81bfe42898e92490dcd2dea3186de72d 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-12-06T16:00:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:36,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/cde848b722cf4da98fd5959d859b98ab is 50, key is test_row_0/A:col10/1733500836536/Put/seqid=0 2024-12-06T16:00:36,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:36,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500896674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:36,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:36,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:36,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500896682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:36,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:36,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:36,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500896684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:36,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741846_1022 (size=14341) 2024-12-06T16:00:36,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:36,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500896686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:36,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/cde848b722cf4da98fd5959d859b98ab 2024-12-06T16:00:36,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:36,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:36,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500896693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:36,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:36,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/24e3bcf1530846458a163e8167fdcdea is 50, key is test_row_0/B:col10/1733500836536/Put/seqid=0 2024-12-06T16:00:36,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741847_1023 (size=12001) 2024-12-06T16:00:36,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500896802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:36,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:36,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500896813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:36,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:36,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500896814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:36,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:36,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500896814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:36,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:36,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:36,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:36,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500896814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:37,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:37,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500897022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:37,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:37,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:37,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500897023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:37,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:37,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:37,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:37,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500897023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:37,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:37,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:37,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:37,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:37,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:37,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:37,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:37,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500897023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:37,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:37,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:37,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:37,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:37,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500897032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:37,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:37,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:37,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:37,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/24e3bcf1530846458a163e8167fdcdea 2024-12-06T16:00:37,189 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T16:00:37,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/f37ed2b1c7f142deab2b201258c79592 is 50, key is test_row_0/C:col10/1733500836536/Put/seqid=0 2024-12-06T16:00:37,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741848_1024 (size=12001) 2024-12-06T16:00:37,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500897332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:37,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:37,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:37,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:37,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500897332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:37,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:37,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:37,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:37,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500897333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:37,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500897333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:37,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:37,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:37,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:37,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:37,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:37,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:37,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:37,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500897344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:37,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:37,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:37,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:37,367 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-06T16:00:37,368 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-06T16:00:37,370 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-06T16:00:37,370 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-06T16:00:37,371 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-06T16:00:37,371 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-06T16:00:37,373 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-06T16:00:37,373 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-06T16:00:37,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/f37ed2b1c7f142deab2b201258c79592 2024-12-06T16:00:37,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/cde848b722cf4da98fd5959d859b98ab as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/cde848b722cf4da98fd5959d859b98ab 2024-12-06T16:00:37,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/cde848b722cf4da98fd5959d859b98ab, entries=200, sequenceid=79, filesize=14.0 K 2024-12-06T16:00:37,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/24e3bcf1530846458a163e8167fdcdea as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/24e3bcf1530846458a163e8167fdcdea 2024-12-06T16:00:37,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/24e3bcf1530846458a163e8167fdcdea, entries=150, sequenceid=79, filesize=11.7 K 2024-12-06T16:00:37,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/f37ed2b1c7f142deab2b201258c79592 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/f37ed2b1c7f142deab2b201258c79592 2024-12-06T16:00:37,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/f37ed2b1c7f142deab2b201258c79592, entries=150, sequenceid=79, filesize=11.7 K 2024-12-06T16:00:37,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=33.54 KB/34350 for 81bfe42898e92490dcd2dea3186de72d in 1081ms, sequenceid=79, compaction requested=true 2024-12-06T16:00:37,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:37,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:00:37,753 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T16:00:37,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:37,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:00:37,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:37,755 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T16:00:37,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:00:37,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:00:37,759 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T16:00:37,761 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/B is initiating minor compaction (all files) 2024-12-06T16:00:37,762 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/B in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:37,762 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/d3b71cfa7747430e93c2dfce60fa05ee, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/20b7171476974be3afffa1704fa2efe8, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/06dbeaedd7054946a34244e746431f4f, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/24e3bcf1530846458a163e8167fdcdea] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=46.9 K 2024-12-06T16:00:37,764 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57364 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T16:00:37,764 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting d3b71cfa7747430e93c2dfce60fa05ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733500834509 2024-12-06T16:00:37,764 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/A is initiating minor compaction (all files) 2024-12-06T16:00:37,764 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/A in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:37,764 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/9c86b711d19445ea90d0dac966b7547d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/2ad7b5dd0a7e4bdca8ff4cfe3a99ce73, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/c1437c485fa648a6a170ea8ad91d4d0b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/cde848b722cf4da98fd5959d859b98ab] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=56.0 K 2024-12-06T16:00:37,765 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c86b711d19445ea90d0dac966b7547d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733500834509 2024-12-06T16:00:37,765 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 20b7171476974be3afffa1704fa2efe8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1733500834774 2024-12-06T16:00:37,767 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ad7b5dd0a7e4bdca8ff4cfe3a99ce73, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1733500834767 2024-12-06T16:00:37,768 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 06dbeaedd7054946a34244e746431f4f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733500836030 2024-12-06T16:00:37,768 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1437c485fa648a6a170ea8ad91d4d0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733500836030 2024-12-06T16:00:37,769 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 24e3bcf1530846458a163e8167fdcdea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733500836527 2024-12-06T16:00:37,769 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting cde848b722cf4da98fd5959d859b98ab, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733500836527 2024-12-06T16:00:37,830 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#A#compaction#13 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:37,830 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#B#compaction#12 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:37,834 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/30ea301006064d6fb56491015907ad61 is 50, key is test_row_0/A:col10/1733500836536/Put/seqid=0 2024-12-06T16:00:37,835 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/abd3c53799c946e993968783d50bf804 is 50, key is test_row_0/B:col10/1733500836536/Put/seqid=0 2024-12-06T16:00:37,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741849_1025 (size=12139) 2024-12-06T16:00:37,870 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:00:37,870 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81bfe42898e92490dcd2dea3186de72d 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-06T16:00:37,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:37,884 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/30ea301006064d6fb56491015907ad61 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/30ea301006064d6fb56491015907ad61 2024-12-06T16:00:37,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741850_1026 (size=12139) 2024-12-06T16:00:37,908 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/abd3c53799c946e993968783d50bf804 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/abd3c53799c946e993968783d50bf804 2024-12-06T16:00:37,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/cb991b5891194564b94ba2d125c01529 is 50, key is test_row_0/A:col10/1733500837866/Put/seqid=0 2024-12-06T16:00:37,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741851_1027 (size=12001) 2024-12-06T16:00:37,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/cb991b5891194564b94ba2d125c01529 2024-12-06T16:00:37,946 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/A of 81bfe42898e92490dcd2dea3186de72d into 30ea301006064d6fb56491015907ad61(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:37,946 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/B of 81bfe42898e92490dcd2dea3186de72d into abd3c53799c946e993968783d50bf804(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:37,946 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:37,946 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:37,950 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/A, priority=12, startTime=1733500837742; duration=0sec 2024-12-06T16:00:37,950 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/B, priority=12, startTime=1733500837754; duration=0sec 2024-12-06T16:00:37,950 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:00:37,951 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:A 2024-12-06T16:00:37,951 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T16:00:37,951 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:37,951 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:B 2024-12-06T16:00:37,956 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T16:00:37,956 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/C is initiating minor compaction (all files) 2024-12-06T16:00:37,956 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/C in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:37,957 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/780815a6351a4b848606dcdc5d037987, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/2b86102c03074554825a233e07750076, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/18c416650918464da4bba863a49fcee3, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/f37ed2b1c7f142deab2b201258c79592] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=46.9 K 2024-12-06T16:00:37,957 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 780815a6351a4b848606dcdc5d037987, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733500834509 2024-12-06T16:00:37,958 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b86102c03074554825a233e07750076, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1733500834774 2024-12-06T16:00:37,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/039ff872c56a45f5b20096f94c86b698 is 50, key is test_row_0/B:col10/1733500837866/Put/seqid=0 2024-12-06T16:00:37,959 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18c416650918464da4bba863a49fcee3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733500836030 2024-12-06T16:00:37,960 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting f37ed2b1c7f142deab2b201258c79592, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733500836527 2024-12-06T16:00:37,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741852_1028 (size=12001) 2024-12-06T16:00:37,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/039ff872c56a45f5b20096f94c86b698 2024-12-06T16:00:38,001 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#C#compaction#16 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:38,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:38,002 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/423a6cc5feee435cba032de14b69c48c is 50, key is test_row_0/C:col10/1733500836536/Put/seqid=0 2024-12-06T16:00:38,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500897981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:38,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:38,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:38,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500897981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:38,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:38,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:38,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500897986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:38,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:38,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:38,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500897987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:38,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:38,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:38,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500897997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:38,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:38,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/7fa9981806254bbe8ab1f7eaf3cb064d is 50, key is test_row_0/C:col10/1733500837866/Put/seqid=0 2024-12-06T16:00:38,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741853_1029 (size=12139) 2024-12-06T16:00:38,048 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/423a6cc5feee435cba032de14b69c48c as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/423a6cc5feee435cba032de14b69c48c 2024-12-06T16:00:38,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741854_1030 (size=12001) 2024-12-06T16:00:38,073 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/C of 81bfe42898e92490dcd2dea3186de72d into 423a6cc5feee435cba032de14b69c48c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:38,073 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:38,074 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/C, priority=12, startTime=1733500837755; duration=0sec 2024-12-06T16:00:38,074 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:38,074 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:C 2024-12-06T16:00:38,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:38,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500898113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:38,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:38,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:38,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500898113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:38,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:38,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500898113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:38,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:38,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:38,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500898113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:38,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:38,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:38,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:38,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500898132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:38,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:38,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:38,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500898323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:38,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:38,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500898323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:38,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:38,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:38,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:38,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500898323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:38,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:38,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500898323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:38,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:38,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:38,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:38,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500898342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:38,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:38,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/7fa9981806254bbe8ab1f7eaf3cb064d 2024-12-06T16:00:38,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/cb991b5891194564b94ba2d125c01529 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/cb991b5891194564b94ba2d125c01529 2024-12-06T16:00:38,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/cb991b5891194564b94ba2d125c01529, entries=150, sequenceid=91, filesize=11.7 K 2024-12-06T16:00:38,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/039ff872c56a45f5b20096f94c86b698 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/039ff872c56a45f5b20096f94c86b698 2024-12-06T16:00:38,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/039ff872c56a45f5b20096f94c86b698, entries=150, sequenceid=91, filesize=11.7 K 2024-12-06T16:00:38,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/7fa9981806254bbe8ab1f7eaf3cb064d as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/7fa9981806254bbe8ab1f7eaf3cb064d 2024-12-06T16:00:38,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/7fa9981806254bbe8ab1f7eaf3cb064d, entries=150, sequenceid=91, filesize=11.7 K 2024-12-06T16:00:38,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 81bfe42898e92490dcd2dea3186de72d in 698ms, sequenceid=91, compaction requested=false 2024-12-06T16:00:38,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:38,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-06T16:00:38,582 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:00:38,583 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:00:38,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-06T16:00:38,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-06T16:00:38,589 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:00:38,590 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:00:38,590 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:00:38,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:38,636 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:00:38,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81bfe42898e92490dcd2dea3186de72d 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-06T16:00:38,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/e1452d8fd06c460e877240a0272ad975 is 50, key is test_row_0/A:col10/1733500838633/Put/seqid=0 2024-12-06T16:00:38,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:38,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500898661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:38,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:38,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500898667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:38,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:38,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500898667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:38,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:38,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500898670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:38,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:38,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500898670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:38,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741855_1031 (size=12001) 2024-12-06T16:00:38,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-06T16:00:38,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-06T16:00:38,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:38,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:38,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:38,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:38,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:38,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:38,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500898782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:38,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:38,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500898783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500898784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:38,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500898784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:38,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500898784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:38,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:38,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:38,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:38,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-06T16:00:38,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-06T16:00:38,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:38,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:38,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:38,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:38,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:38,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:38,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:38,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500898991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:38,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:38,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:38,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500898992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:38,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:38,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500898992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:38,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:38,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:38,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500898992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:38,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:38,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500898992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:38,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:39,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-06T16:00:39,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:39,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:39,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:39,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:39,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:39,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:39,088 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/e1452d8fd06c460e877240a0272ad975 2024-12-06T16:00:39,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/e1a39ae9b9df408da3e49e0aa600badf is 50, key is test_row_0/B:col10/1733500838633/Put/seqid=0 2024-12-06T16:00:39,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741856_1032 (size=12001) 2024-12-06T16:00:39,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/e1a39ae9b9df408da3e49e0aa600badf 2024-12-06T16:00:39,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/ce33412f769c488ea6d790e537c2db76 is 50, key is test_row_0/C:col10/1733500838633/Put/seqid=0 2024-12-06T16:00:39,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-06T16:00:39,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-06T16:00:39,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:39,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:39,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:39,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:39,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:39,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741857_1033 (size=12001) 2024-12-06T16:00:39,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:39,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:39,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500899302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:39,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500899302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:39,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:39,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:39,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:39,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:39,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500899303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:39,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500899302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:39,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500899312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:39,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:39,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:39,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:39,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-06T16:00:39,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:39,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:39,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:39,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:39,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:39,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:39,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-06T16:00:39,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:39,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:39,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:39,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:39,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:39,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:39,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/ce33412f769c488ea6d790e537c2db76 2024-12-06T16:00:39,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/e1452d8fd06c460e877240a0272ad975 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/e1452d8fd06c460e877240a0272ad975 2024-12-06T16:00:39,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/e1452d8fd06c460e877240a0272ad975, entries=150, sequenceid=120, filesize=11.7 K 2024-12-06T16:00:39,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/e1a39ae9b9df408da3e49e0aa600badf as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/e1a39ae9b9df408da3e49e0aa600badf 2024-12-06T16:00:39,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/e1a39ae9b9df408da3e49e0aa600badf, entries=150, sequenceid=120, filesize=11.7 K 2024-12-06T16:00:39,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/ce33412f769c488ea6d790e537c2db76 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/ce33412f769c488ea6d790e537c2db76 2024-12-06T16:00:39,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-06T16:00:39,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:39,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:39,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:39,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:39,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:39,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:39,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/ce33412f769c488ea6d790e537c2db76, entries=150, sequenceid=120, filesize=11.7 K 2024-12-06T16:00:39,693 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=60.38 KB/61830 for 81bfe42898e92490dcd2dea3186de72d in 1057ms, sequenceid=120, compaction requested=true 2024-12-06T16:00:39,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:39,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:00:39,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:39,694 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:00:39,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:00:39,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:00:39,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:00:39,695 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:00:39,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:00:39,698 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:00:39,698 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:00:39,698 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/B is initiating minor compaction (all files) 2024-12-06T16:00:39,698 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/A is initiating minor compaction (all files) 2024-12-06T16:00:39,698 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/B in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:39,698 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/A in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:39,699 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/abd3c53799c946e993968783d50bf804, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/039ff872c56a45f5b20096f94c86b698, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/e1a39ae9b9df408da3e49e0aa600badf] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=35.3 K 2024-12-06T16:00:39,699 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/30ea301006064d6fb56491015907ad61, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/cb991b5891194564b94ba2d125c01529, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/e1452d8fd06c460e877240a0272ad975] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=35.3 K 2024-12-06T16:00:39,699 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting abd3c53799c946e993968783d50bf804, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733500836527 2024-12-06T16:00:39,700 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30ea301006064d6fb56491015907ad61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733500836527 2024-12-06T16:00:39,700 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 039ff872c56a45f5b20096f94c86b698, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733500837860 2024-12-06T16:00:39,701 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb991b5891194564b94ba2d125c01529, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733500837860 2024-12-06T16:00:39,701 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1452d8fd06c460e877240a0272ad975, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733500837953 2024-12-06T16:00:39,701 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting e1a39ae9b9df408da3e49e0aa600badf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733500837953 2024-12-06T16:00:39,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-06T16:00:39,731 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#B#compaction#21 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:39,732 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/5e7b4c5e88774e3c93d47372c963113b is 50, key is test_row_0/B:col10/1733500838633/Put/seqid=0 2024-12-06T16:00:39,738 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#A#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:39,739 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/41b21c1c0f1b4f72ae2efc2cc1c613bb is 50, key is test_row_0/A:col10/1733500838633/Put/seqid=0 2024-12-06T16:00:39,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741858_1034 (size=12241) 2024-12-06T16:00:39,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741859_1035 (size=12241) 2024-12-06T16:00:39,785 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/5e7b4c5e88774e3c93d47372c963113b as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/5e7b4c5e88774e3c93d47372c963113b 2024-12-06T16:00:39,789 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/41b21c1c0f1b4f72ae2efc2cc1c613bb as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/41b21c1c0f1b4f72ae2efc2cc1c613bb 2024-12-06T16:00:39,808 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/B of 81bfe42898e92490dcd2dea3186de72d into 5e7b4c5e88774e3c93d47372c963113b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:39,809 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:39,809 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/B, priority=13, startTime=1733500839695; duration=0sec 2024-12-06T16:00:39,809 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:00:39,809 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:B 2024-12-06T16:00:39,809 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:00:39,810 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/A of 81bfe42898e92490dcd2dea3186de72d into 41b21c1c0f1b4f72ae2efc2cc1c613bb(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:39,810 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:39,810 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/A, priority=13, startTime=1733500839694; duration=0sec 2024-12-06T16:00:39,811 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:39,811 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:A 2024-12-06T16:00:39,816 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:00:39,816 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/C is initiating minor compaction (all files) 2024-12-06T16:00:39,816 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/C in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:39,817 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/423a6cc5feee435cba032de14b69c48c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/7fa9981806254bbe8ab1f7eaf3cb064d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/ce33412f769c488ea6d790e537c2db76] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=35.3 K 2024-12-06T16:00:39,818 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 423a6cc5feee435cba032de14b69c48c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733500836527 2024-12-06T16:00:39,818 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:00:39,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81bfe42898e92490dcd2dea3186de72d 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-06T16:00:39,819 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fa9981806254bbe8ab1f7eaf3cb064d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733500837860 2024-12-06T16:00:39,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:39,822 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting ce33412f769c488ea6d790e537c2db76, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733500837953 2024-12-06T16:00:39,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/f262ded6286847fdbf7fb88bdaa5e4f7 is 50, key is test_row_0/A:col10/1733500838666/Put/seqid=0 2024-12-06T16:00:39,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-06T16:00:39,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:39,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:39,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:39,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:39,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:39,856 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#C#compaction#24 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:39,857 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/698244ac2ca945e5be2ec6590800a8e3 is 50, key is test_row_0/C:col10/1733500838633/Put/seqid=0 2024-12-06T16:00:39,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741860_1036 (size=12151) 2024-12-06T16:00:39,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/f262ded6286847fdbf7fb88bdaa5e4f7 2024-12-06T16:00:39,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741861_1037 (size=12241) 2024-12-06T16:00:39,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500899885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:39,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500899885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:39,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:39,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:39,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:39,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:39,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:39,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:39,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500899897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:39,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:39,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:39,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:39,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500899897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:39,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:39,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:39,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:39,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500899898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:39,915 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/698244ac2ca945e5be2ec6590800a8e3 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/698244ac2ca945e5be2ec6590800a8e3 2024-12-06T16:00:39,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/7cb03216a4e244b4826692ff0485fc20 is 50, key is test_row_0/B:col10/1733500838666/Put/seqid=0 2024-12-06T16:00:39,939 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/C of 81bfe42898e92490dcd2dea3186de72d into 698244ac2ca945e5be2ec6590800a8e3(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:39,939 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:39,939 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/C, priority=13, startTime=1733500839695; duration=0sec 2024-12-06T16:00:39,939 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:39,939 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:C 2024-12-06T16:00:39,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741862_1038 (size=12151) 2024-12-06T16:00:39,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/7cb03216a4e244b4826692ff0485fc20 2024-12-06T16:00:39,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/cbfd028b010b4a918b7d92182b2a3206 is 50, key is test_row_0/C:col10/1733500838666/Put/seqid=0 2024-12-06T16:00:39,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741863_1039 (size=12151) 2024-12-06T16:00:39,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/cbfd028b010b4a918b7d92182b2a3206 2024-12-06T16:00:39,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-06T16:00:39,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:39,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:39,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:39,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:39,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:40,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500900002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:40,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:40,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/f262ded6286847fdbf7fb88bdaa5e4f7 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/f262ded6286847fdbf7fb88bdaa5e4f7 2024-12-06T16:00:40,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500900006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:40,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:40,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500900013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:40,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:40,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/f262ded6286847fdbf7fb88bdaa5e4f7, entries=150, sequenceid=135, filesize=11.9 K 2024-12-06T16:00:40,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500900013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:40,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:40,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/7cb03216a4e244b4826692ff0485fc20 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/7cb03216a4e244b4826692ff0485fc20 2024-12-06T16:00:40,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:40,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500900023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:40,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:40,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/7cb03216a4e244b4826692ff0485fc20, entries=150, sequenceid=135, filesize=11.9 K 2024-12-06T16:00:40,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/cbfd028b010b4a918b7d92182b2a3206 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/cbfd028b010b4a918b7d92182b2a3206 2024-12-06T16:00:40,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/cbfd028b010b4a918b7d92182b2a3206, entries=150, sequenceid=135, filesize=11.9 K 2024-12-06T16:00:40,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 81bfe42898e92490dcd2dea3186de72d in 234ms, sequenceid=135, compaction requested=false 2024-12-06T16:00:40,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:40,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-06T16:00:40,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:40,154 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 81bfe42898e92490dcd2dea3186de72d 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-06T16:00:40,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/b484b8d1e1cb40cf81169bd0821dff04 is 50, key is test_row_0/A:col10/1733500839888/Put/seqid=0 2024-12-06T16:00:40,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741864_1040 (size=12151) 2024-12-06T16:00:40,208 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/b484b8d1e1cb40cf81169bd0821dff04 2024-12-06T16:00:40,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:40,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:40,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/98d0cd4405c94303b4ec0668a3f4f0c2 is 50, key is test_row_0/B:col10/1733500839888/Put/seqid=0 2024-12-06T16:00:40,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:40,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500900245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:40,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:40,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:40,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500900251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:40,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:40,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:40,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500900253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:40,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:40,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:40,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500900261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:40,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:40,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:40,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500900261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:40,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:40,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741865_1041 (size=12151) 2024-12-06T16:00:40,278 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/98d0cd4405c94303b4ec0668a3f4f0c2 2024-12-06T16:00:40,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/79a5c2e7c0594e9d82535d8aff691a60 is 50, key is test_row_0/C:col10/1733500839888/Put/seqid=0 2024-12-06T16:00:40,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741866_1042 (size=12151) 2024-12-06T16:00:40,328 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/79a5c2e7c0594e9d82535d8aff691a60 2024-12-06T16:00:40,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/b484b8d1e1cb40cf81169bd0821dff04 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/b484b8d1e1cb40cf81169bd0821dff04 2024-12-06T16:00:40,360 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/b484b8d1e1cb40cf81169bd0821dff04, entries=150, sequenceid=160, filesize=11.9 K 2024-12-06T16:00:40,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/98d0cd4405c94303b4ec0668a3f4f0c2 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/98d0cd4405c94303b4ec0668a3f4f0c2 2024-12-06T16:00:40,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500900362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:40,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:40,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:40,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:40,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500900363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:40,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500900362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:40,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:40,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:40,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:40,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500900372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:40,378 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/98d0cd4405c94303b4ec0668a3f4f0c2, entries=150, sequenceid=160, filesize=11.9 K 2024-12-06T16:00:40,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:40,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/79a5c2e7c0594e9d82535d8aff691a60 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/79a5c2e7c0594e9d82535d8aff691a60 2024-12-06T16:00:40,388 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/79a5c2e7c0594e9d82535d8aff691a60, entries=150, sequenceid=160, filesize=11.9 K 2024-12-06T16:00:40,391 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 81bfe42898e92490dcd2dea3186de72d in 237ms, sequenceid=160, compaction requested=true 2024-12-06T16:00:40,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:40,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:40,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-06T16:00:40,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-06T16:00:40,397 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-06T16:00:40,397 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8040 sec 2024-12-06T16:00:40,401 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.8160 sec 2024-12-06T16:00:40,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:40,580 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:00:40,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81bfe42898e92490dcd2dea3186de72d 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-06T16:00:40,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/265e8fa92c4d4dcda47570cff1f05208 is 50, key is test_row_0/A:col10/1733500840242/Put/seqid=0 2024-12-06T16:00:40,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741867_1043 (size=12151) 2024-12-06T16:00:40,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/265e8fa92c4d4dcda47570cff1f05208 2024-12-06T16:00:40,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/8e66120fe00e4368a81a4a665c9d3271 is 50, key is test_row_0/B:col10/1733500840242/Put/seqid=0 2024-12-06T16:00:40,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:40,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500900629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:40,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:40,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:40,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500900631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:40,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:40,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:40,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500900632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:40,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:40,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:40,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500900634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:40,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:40,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:40,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500900641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:40,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:40,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741868_1044 (size=12151) 2024-12-06T16:00:40,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/8e66120fe00e4368a81a4a665c9d3271 2024-12-06T16:00:40,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/c95173f4df9b4784a49552aa6fd47bf3 is 50, key is test_row_0/C:col10/1733500840242/Put/seqid=0 2024-12-06T16:00:40,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741869_1045 (size=12151) 2024-12-06T16:00:40,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-06T16:00:40,736 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:00:40,737 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:00:40,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-06T16:00:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-06T16:00:40,742 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:00:40,744 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:00:40,745 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:00:40,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:40,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500900752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:40,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500900753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:40,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:40,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:40,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500900753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:40,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:40,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:40,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500900753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:40,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:40,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:40,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500900754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:40,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:40,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-06T16:00:40,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-06T16:00:40,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:40,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:40,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:40,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:40,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:40,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:40,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:40,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500900962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:40,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:40,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:40,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500900972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:40,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:40,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:40,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500900972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:40,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:40,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500900972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:40,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:40,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500900972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:40,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:40,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:40,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:40,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:41,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-06T16:00:41,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:41,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:41,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:41,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:41,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:41,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:41,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-06T16:00:41,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/c95173f4df9b4784a49552aa6fd47bf3 2024-12-06T16:00:41,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/265e8fa92c4d4dcda47570cff1f05208 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/265e8fa92c4d4dcda47570cff1f05208 2024-12-06T16:00:41,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/265e8fa92c4d4dcda47570cff1f05208, entries=150, sequenceid=173, filesize=11.9 K 2024-12-06T16:00:41,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/8e66120fe00e4368a81a4a665c9d3271 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/8e66120fe00e4368a81a4a665c9d3271 2024-12-06T16:00:41,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/8e66120fe00e4368a81a4a665c9d3271, entries=150, sequenceid=173, filesize=11.9 K 2024-12-06T16:00:41,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/c95173f4df9b4784a49552aa6fd47bf3 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/c95173f4df9b4784a49552aa6fd47bf3 2024-12-06T16:00:41,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/c95173f4df9b4784a49552aa6fd47bf3, entries=150, sequenceid=173, filesize=11.9 K 2024-12-06T16:00:41,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 81bfe42898e92490dcd2dea3186de72d in 583ms, sequenceid=173, compaction requested=true 2024-12-06T16:00:41,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:41,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:00:41,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:41,164 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T16:00:41,164 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T16:00:41,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:00:41,167 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T16:00:41,167 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T16:00:41,167 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/A is initiating minor compaction (all files) 2024-12-06T16:00:41,167 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/B is initiating minor compaction (all files) 2024-12-06T16:00:41,167 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/A in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:41,167 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/B in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:41,168 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/5e7b4c5e88774e3c93d47372c963113b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/7cb03216a4e244b4826692ff0485fc20, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/98d0cd4405c94303b4ec0668a3f4f0c2, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/8e66120fe00e4368a81a4a665c9d3271] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=47.6 K 2024-12-06T16:00:41,168 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/41b21c1c0f1b4f72ae2efc2cc1c613bb, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/f262ded6286847fdbf7fb88bdaa5e4f7, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/b484b8d1e1cb40cf81169bd0821dff04, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/265e8fa92c4d4dcda47570cff1f05208] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=47.6 K 2024-12-06T16:00:41,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:41,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:00:41,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:00:41,169 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e7b4c5e88774e3c93d47372c963113b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733500837953 2024-12-06T16:00:41,169 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41b21c1c0f1b4f72ae2efc2cc1c613bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733500837953 2024-12-06T16:00:41,171 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting f262ded6286847fdbf7fb88bdaa5e4f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733500838664 2024-12-06T16:00:41,172 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cb03216a4e244b4826692ff0485fc20, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733500838664 2024-12-06T16:00:41,172 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 98d0cd4405c94303b4ec0668a3f4f0c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733500839888 2024-12-06T16:00:41,173 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting b484b8d1e1cb40cf81169bd0821dff04, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733500839888 2024-12-06T16:00:41,173 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e66120fe00e4368a81a4a665c9d3271, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733500840221 2024-12-06T16:00:41,175 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 265e8fa92c4d4dcda47570cff1f05208, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733500840221 2024-12-06T16:00:41,196 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#B#compaction#33 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:41,197 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/99e24d2fdf5142d5a419529da13439bc is 50, key is test_row_0/B:col10/1733500840242/Put/seqid=0 2024-12-06T16:00:41,199 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#A#compaction#34 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:41,199 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/3896fd23dd4b4c4e96773db34dfb649f is 50, key is test_row_0/A:col10/1733500840242/Put/seqid=0 2024-12-06T16:00:41,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-06T16:00:41,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:41,213 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 81bfe42898e92490dcd2dea3186de72d 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-06T16:00:41,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741870_1046 (size=12527) 2024-12-06T16:00:41,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/6f9fb8e2999943078f59e748f5bd95c6 is 50, key is test_row_0/A:col10/1733500840629/Put/seqid=0 2024-12-06T16:00:41,254 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/99e24d2fdf5142d5a419529da13439bc as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/99e24d2fdf5142d5a419529da13439bc 2024-12-06T16:00:41,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741871_1047 (size=12527) 2024-12-06T16:00:41,268 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/B of 81bfe42898e92490dcd2dea3186de72d into 99e24d2fdf5142d5a419529da13439bc(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:41,268 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:41,268 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/B, priority=12, startTime=1733500841164; duration=0sec 2024-12-06T16:00:41,268 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:00:41,268 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:B 2024-12-06T16:00:41,269 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T16:00:41,271 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T16:00:41,272 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/C is initiating minor compaction (all files) 2024-12-06T16:00:41,274 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/C in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:41,275 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/698244ac2ca945e5be2ec6590800a8e3, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/cbfd028b010b4a918b7d92182b2a3206, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/79a5c2e7c0594e9d82535d8aff691a60, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/c95173f4df9b4784a49552aa6fd47bf3] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=47.6 K 2024-12-06T16:00:41,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:41,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:41,280 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 698244ac2ca945e5be2ec6590800a8e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733500837953 2024-12-06T16:00:41,280 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/3896fd23dd4b4c4e96773db34dfb649f as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/3896fd23dd4b4c4e96773db34dfb649f 2024-12-06T16:00:41,284 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting cbfd028b010b4a918b7d92182b2a3206, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733500838664 2024-12-06T16:00:41,286 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 79a5c2e7c0594e9d82535d8aff691a60, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733500839888 2024-12-06T16:00:41,288 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting c95173f4df9b4784a49552aa6fd47bf3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733500840221 2024-12-06T16:00:41,291 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/A of 81bfe42898e92490dcd2dea3186de72d into 3896fd23dd4b4c4e96773db34dfb649f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:41,291 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:41,291 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/A, priority=12, startTime=1733500841164; duration=0sec 2024-12-06T16:00:41,291 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:41,291 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:A 2024-12-06T16:00:41,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741872_1048 (size=12151) 2024-12-06T16:00:41,299 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/6f9fb8e2999943078f59e748f5bd95c6 2024-12-06T16:00:41,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/474f5f44b33a48cf9d76b4ee7612d6bb is 50, key is test_row_0/B:col10/1733500840629/Put/seqid=0 2024-12-06T16:00:41,324 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#C#compaction#37 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:41,325 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/84125e8896524524a9ee49d6ef749d10 is 50, key is test_row_0/C:col10/1733500840242/Put/seqid=0 2024-12-06T16:00:41,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741873_1049 (size=12151) 2024-12-06T16:00:41,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:41,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500901317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:41,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:41,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500901321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:41,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:41,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:41,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741874_1050 (size=12527) 2024-12-06T16:00:41,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500901342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:41,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:41,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-06T16:00:41,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:41,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500901364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:41,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:41,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:41,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500901364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:41,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:41,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:41,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:41,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500901453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:41,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500901452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:41,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:41,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:41,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:41,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500901472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:41,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:41,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:41,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500901482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:41,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:41,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500901483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:41,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:41,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:41,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:41,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500901662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:41,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500901662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:41,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:41,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:41,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500901682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:41,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:41,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:41,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500901692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:41,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:41,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:41,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500901692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:41,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:41,740 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/474f5f44b33a48cf9d76b4ee7612d6bb 2024-12-06T16:00:41,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/52eea21c14fd4bc9ad27b6c944029622 is 50, key is test_row_0/C:col10/1733500840629/Put/seqid=0 2024-12-06T16:00:41,762 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/84125e8896524524a9ee49d6ef749d10 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/84125e8896524524a9ee49d6ef749d10 2024-12-06T16:00:41,771 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/C of 81bfe42898e92490dcd2dea3186de72d into 84125e8896524524a9ee49d6ef749d10(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:41,772 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:41,772 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/C, priority=12, startTime=1733500841168; duration=0sec 2024-12-06T16:00:41,772 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:41,772 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:C 2024-12-06T16:00:41,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741875_1051 (size=12151) 2024-12-06T16:00:41,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-06T16:00:41,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:41,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500901972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:41,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:41,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:41,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500901973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:41,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:41,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:41,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500901992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:41,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:41,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:42,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:42,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500902002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:42,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:42,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:42,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500902003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:42,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:42,175 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/52eea21c14fd4bc9ad27b6c944029622 2024-12-06T16:00:42,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/6f9fb8e2999943078f59e748f5bd95c6 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/6f9fb8e2999943078f59e748f5bd95c6 2024-12-06T16:00:42,192 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/6f9fb8e2999943078f59e748f5bd95c6, entries=150, sequenceid=196, filesize=11.9 K 2024-12-06T16:00:42,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/474f5f44b33a48cf9d76b4ee7612d6bb as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/474f5f44b33a48cf9d76b4ee7612d6bb 2024-12-06T16:00:42,223 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/474f5f44b33a48cf9d76b4ee7612d6bb, entries=150, sequenceid=196, filesize=11.9 K 2024-12-06T16:00:42,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/52eea21c14fd4bc9ad27b6c944029622 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/52eea21c14fd4bc9ad27b6c944029622 2024-12-06T16:00:42,246 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/52eea21c14fd4bc9ad27b6c944029622, entries=150, sequenceid=196, filesize=11.9 K 2024-12-06T16:00:42,247 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=80.51 KB/82440 for 81bfe42898e92490dcd2dea3186de72d in 1034ms, sequenceid=196, compaction requested=false 2024-12-06T16:00:42,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:42,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:42,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-06T16:00:42,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-06T16:00:42,258 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-06T16:00:42,258 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5100 sec 2024-12-06T16:00:42,263 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.5220 sec 2024-12-06T16:00:42,486 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:00:42,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:42,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81bfe42898e92490dcd2dea3186de72d 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-06T16:00:42,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/a2a9179d493b4963b3ad249951f28f4e is 50, key is test_row_0/A:col10/1733500842484/Put/seqid=0 2024-12-06T16:00:42,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741876_1052 (size=16931) 2024-12-06T16:00:42,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/a2a9179d493b4963b3ad249951f28f4e 2024-12-06T16:00:42,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:42,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:42,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500902530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:42,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500902532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:42,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:42,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500902533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:42,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:42,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,546 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:42,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:42,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500902540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:42,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:42,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:42,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500902545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:42,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:42,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/017203f6d03c4d3ca4325eed201e0105 is 50, key is test_row_0/B:col10/1733500842484/Put/seqid=0 2024-12-06T16:00:42,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741877_1053 (size=12151) 2024-12-06T16:00:42,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500902652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:42,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:42,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500902652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:42,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:42,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500902652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500902652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:42,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500902653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:42,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:42,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500902861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:42,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:42,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:42,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500902862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:42,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:42,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:42,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500902862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:42,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:42,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500902862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:42,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:42,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:42,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500902862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:42,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-06T16:00:42,892 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:00:42,893 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:00:42,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-06T16:00:42,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-06T16:00:42,896 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:00:42,897 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:00:42,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:00:42,969 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/017203f6d03c4d3ca4325eed201e0105 2024-12-06T16:00:42,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/c0c937516075446085bd9d21c2cc7d0b is 50, key is test_row_0/C:col10/1733500842484/Put/seqid=0 2024-12-06T16:00:42,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741878_1054 (size=12151) 2024-12-06T16:00:42,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/c0c937516075446085bd9d21c2cc7d0b 2024-12-06T16:00:43,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/a2a9179d493b4963b3ad249951f28f4e as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/a2a9179d493b4963b3ad249951f28f4e 2024-12-06T16:00:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-06T16:00:43,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/a2a9179d493b4963b3ad249951f28f4e, entries=250, sequenceid=216, filesize=16.5 K 2024-12-06T16:00:43,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/017203f6d03c4d3ca4325eed201e0105 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/017203f6d03c4d3ca4325eed201e0105 2024-12-06T16:00:43,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/017203f6d03c4d3ca4325eed201e0105, entries=150, sequenceid=216, filesize=11.9 K 2024-12-06T16:00:43,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/c0c937516075446085bd9d21c2cc7d0b as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/c0c937516075446085bd9d21c2cc7d0b 2024-12-06T16:00:43,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/c0c937516075446085bd9d21c2cc7d0b, entries=150, sequenceid=216, filesize=11.9 K 2024-12-06T16:00:43,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 81bfe42898e92490dcd2dea3186de72d in 565ms, sequenceid=216, compaction requested=true 2024-12-06T16:00:43,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:43,051 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:00:43,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:00:43,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:43,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-06T16:00:43,052 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:00:43,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:43,052 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 81bfe42898e92490dcd2dea3186de72d 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-12-06T16:00:43,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:00:43,053 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41609 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:00:43,053 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/A is initiating minor compaction (all files) 2024-12-06T16:00:43,053 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/A in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:43,053 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/3896fd23dd4b4c4e96773db34dfb649f, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/6f9fb8e2999943078f59e748f5bd95c6, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/a2a9179d493b4963b3ad249951f28f4e] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=40.6 K 2024-12-06T16:00:43,054 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3896fd23dd4b4c4e96773db34dfb649f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733500840221 2024-12-06T16:00:43,054 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:00:43,054 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/B is initiating minor compaction (all files) 2024-12-06T16:00:43,054 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/B in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:43,054 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/99e24d2fdf5142d5a419529da13439bc, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/474f5f44b33a48cf9d76b4ee7612d6bb, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/017203f6d03c4d3ca4325eed201e0105] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=36.0 K 2024-12-06T16:00:43,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:43,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:00:43,055 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 99e24d2fdf5142d5a419529da13439bc, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733500840221 2024-12-06T16:00:43,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:00:43,055 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f9fb8e2999943078f59e748f5bd95c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733500840629 2024-12-06T16:00:43,056 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 474f5f44b33a48cf9d76b4ee7612d6bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733500840629 2024-12-06T16:00:43,056 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2a9179d493b4963b3ad249951f28f4e, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733500841308 2024-12-06T16:00:43,059 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 017203f6d03c4d3ca4325eed201e0105, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733500841339 2024-12-06T16:00:43,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/0a24ed42627d42b4af34e6d30f666600 is 50, key is test_row_0/A:col10/1733500842539/Put/seqid=0 2024-12-06T16:00:43,079 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#A#compaction#43 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:43,079 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/68e4f52a7f574aae8aa6e82ecb20b6d8 is 50, key is test_row_0/A:col10/1733500842484/Put/seqid=0 2024-12-06T16:00:43,090 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#B#compaction#44 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:43,091 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/83534f3cd9c04025937fff0b4706d81f is 50, key is test_row_0/B:col10/1733500842484/Put/seqid=0 2024-12-06T16:00:43,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741879_1055 (size=12629) 2024-12-06T16:00:43,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741880_1056 (size=12151) 2024-12-06T16:00:43,137 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/0a24ed42627d42b4af34e6d30f666600 2024-12-06T16:00:43,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741881_1057 (size=12629) 2024-12-06T16:00:43,156 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/83534f3cd9c04025937fff0b4706d81f as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/83534f3cd9c04025937fff0b4706d81f 2024-12-06T16:00:43,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/e0c3fb20c1654f3d99cb527ad014f964 is 50, key is test_row_0/B:col10/1733500842539/Put/seqid=0 2024-12-06T16:00:43,166 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/B of 81bfe42898e92490dcd2dea3186de72d into 83534f3cd9c04025937fff0b4706d81f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:43,166 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:43,166 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/B, priority=13, startTime=1733500843052; duration=0sec 2024-12-06T16:00:43,166 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:00:43,166 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:B 2024-12-06T16:00:43,166 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:00:43,170 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:00:43,170 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/C is initiating minor compaction (all files) 2024-12-06T16:00:43,170 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/C in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:43,170 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/84125e8896524524a9ee49d6ef749d10, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/52eea21c14fd4bc9ad27b6c944029622, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/c0c937516075446085bd9d21c2cc7d0b] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=36.0 K 2024-12-06T16:00:43,171 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 84125e8896524524a9ee49d6ef749d10, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733500840221 2024-12-06T16:00:43,172 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 52eea21c14fd4bc9ad27b6c944029622, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733500840629 2024-12-06T16:00:43,172 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting c0c937516075446085bd9d21c2cc7d0b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733500841339 2024-12-06T16:00:43,185 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#C#compaction#46 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:43,186 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/0f827ce12fa64c96beadfb3595973869 is 50, key is test_row_0/C:col10/1733500842484/Put/seqid=0 2024-12-06T16:00:43,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741882_1058 (size=12151) 2024-12-06T16:00:43,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741883_1059 (size=12629) 2024-12-06T16:00:43,203 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/0f827ce12fa64c96beadfb3595973869 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/0f827ce12fa64c96beadfb3595973869 2024-12-06T16:00:43,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:43,211 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/C of 81bfe42898e92490dcd2dea3186de72d into 0f827ce12fa64c96beadfb3595973869(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:43,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:43,211 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:43,211 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/C, priority=13, startTime=1733500843055; duration=0sec 2024-12-06T16:00:43,211 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:43,211 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:C 2024-12-06T16:00:43,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-06T16:00:43,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:43,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500903240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:43,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:43,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:43,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500903245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:43,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:43,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500903245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:43,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:43,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:43,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:43,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500903249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:43,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:43,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:43,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500903253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:43,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:43,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:43,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500903352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:43,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:43,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:43,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500903361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:43,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:43,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:43,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500903361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:43,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:43,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500903362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:43,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:43,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:43,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500903362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:43,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:43,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:43,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-06T16:00:43,540 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/68e4f52a7f574aae8aa6e82ecb20b6d8 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/68e4f52a7f574aae8aa6e82ecb20b6d8 2024-12-06T16:00:43,549 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/A of 81bfe42898e92490dcd2dea3186de72d into 68e4f52a7f574aae8aa6e82ecb20b6d8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:43,549 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:43,549 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/A, priority=13, startTime=1733500843051; duration=0sec 2024-12-06T16:00:43,549 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:43,549 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:A 2024-12-06T16:00:43,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:43,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500903562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:43,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:43,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:43,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500903573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:43,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:43,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500903573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:43,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:43,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500903574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:43,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:43,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:43,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:43,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:43,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500903575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:43,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:43,589 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/e0c3fb20c1654f3d99cb527ad014f964 2024-12-06T16:00:43,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/8956e325d48f4f2691df11e757efc359 is 50, key is test_row_0/C:col10/1733500842539/Put/seqid=0 2024-12-06T16:00:43,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741884_1060 (size=12151) 2024-12-06T16:00:43,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:43,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500903872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:43,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500903882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:43,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:43,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500903882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:43,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:43,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:43,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500903892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:43,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:43,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500903892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:43,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:43,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:43,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:44,013 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/8956e325d48f4f2691df11e757efc359 2024-12-06T16:00:44,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/0a24ed42627d42b4af34e6d30f666600 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/0a24ed42627d42b4af34e6d30f666600 2024-12-06T16:00:44,031 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/0a24ed42627d42b4af34e6d30f666600, entries=150, sequenceid=235, filesize=11.9 K 2024-12-06T16:00:44,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/e0c3fb20c1654f3d99cb527ad014f964 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/e0c3fb20c1654f3d99cb527ad014f964 2024-12-06T16:00:44,038 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/e0c3fb20c1654f3d99cb527ad014f964, entries=150, sequenceid=235, filesize=11.9 K 2024-12-06T16:00:44,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/8956e325d48f4f2691df11e757efc359 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/8956e325d48f4f2691df11e757efc359 2024-12-06T16:00:44,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-06T16:00:44,047 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/8956e325d48f4f2691df11e757efc359, entries=150, sequenceid=235, filesize=11.9 K 2024-12-06T16:00:44,049 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 81bfe42898e92490dcd2dea3186de72d in 997ms, sequenceid=235, compaction requested=false 2024-12-06T16:00:44,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:44,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:44,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-06T16:00:44,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-06T16:00:44,056 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-06T16:00:44,056 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1540 sec 2024-12-06T16:00:44,059 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.1640 sec 2024-12-06T16:00:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:44,386 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:00:44,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81bfe42898e92490dcd2dea3186de72d 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-12-06T16:00:44,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/721cbb5637a44ae8a7bde7a4a0f0569b is 50, key is test_row_0/A:col10/1733500843242/Put/seqid=0 2024-12-06T16:00:44,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741885_1061 (size=14591) 2024-12-06T16:00:44,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/721cbb5637a44ae8a7bde7a4a0f0569b 2024-12-06T16:00:44,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/371c112a97ac482c950dc4b39fe771c5 is 50, key is test_row_0/B:col10/1733500843242/Put/seqid=0 2024-12-06T16:00:44,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:44,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500904418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:44,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:44,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500904423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:44,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500904423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:44,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:44,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500904426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:44,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500904426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:44,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:44,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:44,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:44,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741886_1062 (size=12201) 2024-12-06T16:00:44,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/371c112a97ac482c950dc4b39fe771c5 2024-12-06T16:00:44,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/40cfd0cc6c8f47809c2953b7b1e4ddc4 is 50, key is test_row_0/C:col10/1733500843242/Put/seqid=0 2024-12-06T16:00:44,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741887_1063 (size=12201) 2024-12-06T16:00:44,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/40cfd0cc6c8f47809c2953b7b1e4ddc4 2024-12-06T16:00:44,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/721cbb5637a44ae8a7bde7a4a0f0569b as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/721cbb5637a44ae8a7bde7a4a0f0569b 2024-12-06T16:00:44,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/721cbb5637a44ae8a7bde7a4a0f0569b, entries=200, sequenceid=257, filesize=14.2 K 2024-12-06T16:00:44,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/371c112a97ac482c950dc4b39fe771c5 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/371c112a97ac482c950dc4b39fe771c5 2024-12-06T16:00:44,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/371c112a97ac482c950dc4b39fe771c5, entries=150, sequenceid=257, filesize=11.9 K 2024-12-06T16:00:44,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/40cfd0cc6c8f47809c2953b7b1e4ddc4 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/40cfd0cc6c8f47809c2953b7b1e4ddc4 2024-12-06T16:00:44,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/40cfd0cc6c8f47809c2953b7b1e4ddc4, entries=150, sequenceid=257, filesize=11.9 K 2024-12-06T16:00:44,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 81bfe42898e92490dcd2dea3186de72d in 142ms, sequenceid=257, compaction requested=true 2024-12-06T16:00:44,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:44,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:00:44,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:44,529 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:00:44,529 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:00:44,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:00:44,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:44,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:00:44,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:00:44,530 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39371 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:00:44,530 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/A is initiating minor compaction (all files) 2024-12-06T16:00:44,530 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36981 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:00:44,531 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/A in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:44,531 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/B is initiating minor compaction (all files) 2024-12-06T16:00:44,531 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/B in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:44,531 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/68e4f52a7f574aae8aa6e82ecb20b6d8, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/0a24ed42627d42b4af34e6d30f666600, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/721cbb5637a44ae8a7bde7a4a0f0569b] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=38.4 K 2024-12-06T16:00:44,531 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/83534f3cd9c04025937fff0b4706d81f, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/e0c3fb20c1654f3d99cb527ad014f964, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/371c112a97ac482c950dc4b39fe771c5] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=36.1 K 2024-12-06T16:00:44,531 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 83534f3cd9c04025937fff0b4706d81f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733500841339 2024-12-06T16:00:44,532 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68e4f52a7f574aae8aa6e82ecb20b6d8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733500841339 2024-12-06T16:00:44,532 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a24ed42627d42b4af34e6d30f666600, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733500842516 2024-12-06T16:00:44,533 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting e0c3fb20c1654f3d99cb527ad014f964, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733500842516 2024-12-06T16:00:44,533 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 721cbb5637a44ae8a7bde7a4a0f0569b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733500843226 2024-12-06T16:00:44,534 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 371c112a97ac482c950dc4b39fe771c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733500843226 2024-12-06T16:00:44,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:44,538 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:00:44,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81bfe42898e92490dcd2dea3186de72d 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-12-06T16:00:44,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/66b953419652443984127f8ea327a8ad is 50, key is test_row_0/A:col10/1733500844421/Put/seqid=0 2024-12-06T16:00:44,554 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#B#compaction#52 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:44,555 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/2ced15e61f184e47a3c714f5dcf6c11e is 50, key is test_row_0/B:col10/1733500843242/Put/seqid=0 2024-12-06T16:00:44,555 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#A#compaction#53 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:44,557 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/1392ee1eb54541e0a91cc0899bfe8c60 is 50, key is test_row_0/A:col10/1733500843242/Put/seqid=0 2024-12-06T16:00:44,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741888_1064 (size=12301) 2024-12-06T16:00:44,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/66b953419652443984127f8ea327a8ad 2024-12-06T16:00:44,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:44,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500904580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:44,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:44,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500904580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:44,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:44,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500904582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:44,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:44,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:44,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:44,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500904583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:44,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:44,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:44,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500904587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:44,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:44,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741889_1065 (size=12781) 2024-12-06T16:00:44,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741890_1066 (size=12781) 2024-12-06T16:00:44,631 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/2ced15e61f184e47a3c714f5dcf6c11e as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/2ced15e61f184e47a3c714f5dcf6c11e 2024-12-06T16:00:44,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/1bf26d526b95454890a854462486d30a is 50, key is test_row_0/B:col10/1733500844421/Put/seqid=0 2024-12-06T16:00:44,635 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/1392ee1eb54541e0a91cc0899bfe8c60 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/1392ee1eb54541e0a91cc0899bfe8c60 2024-12-06T16:00:44,644 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/B of 81bfe42898e92490dcd2dea3186de72d into 2ced15e61f184e47a3c714f5dcf6c11e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:44,644 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:44,644 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/B, priority=13, startTime=1733500844529; duration=0sec 2024-12-06T16:00:44,644 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:00:44,644 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:B 2024-12-06T16:00:44,645 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/A of 81bfe42898e92490dcd2dea3186de72d into 1392ee1eb54541e0a91cc0899bfe8c60(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:44,645 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:00:44,645 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:44,645 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/A, priority=13, startTime=1733500844528; duration=0sec 2024-12-06T16:00:44,645 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:44,645 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:A 2024-12-06T16:00:44,646 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36981 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:00:44,646 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/C is initiating minor compaction (all files) 2024-12-06T16:00:44,646 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/C in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:44,646 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/0f827ce12fa64c96beadfb3595973869, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/8956e325d48f4f2691df11e757efc359, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/40cfd0cc6c8f47809c2953b7b1e4ddc4] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=36.1 K 2024-12-06T16:00:44,649 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f827ce12fa64c96beadfb3595973869, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733500841339 2024-12-06T16:00:44,650 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 8956e325d48f4f2691df11e757efc359, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733500842516 2024-12-06T16:00:44,650 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 40cfd0cc6c8f47809c2953b7b1e4ddc4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733500843226 2024-12-06T16:00:44,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741891_1067 (size=12301) 2024-12-06T16:00:44,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/1bf26d526b95454890a854462486d30a 2024-12-06T16:00:44,675 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#C#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:44,676 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/e8a0a42aeb224ecb826258eb878db948 is 50, key is test_row_0/C:col10/1733500843242/Put/seqid=0 2024-12-06T16:00:44,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/08c8742946864c9f9d6ac5f9ea3ed45f is 50, key is test_row_0/C:col10/1733500844421/Put/seqid=0 2024-12-06T16:00:44,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741892_1068 (size=12781) 2024-12-06T16:00:44,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:44,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500904703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500904703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:44,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:44,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:44,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500904703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:44,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:44,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:44,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741893_1069 (size=12301) 2024-12-06T16:00:44,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/08c8742946864c9f9d6ac5f9ea3ed45f 2024-12-06T16:00:44,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:44,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500904712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:44,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:44,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500904713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:44,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:44,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:44,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/66b953419652443984127f8ea327a8ad as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/66b953419652443984127f8ea327a8ad 2024-12-06T16:00:44,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/66b953419652443984127f8ea327a8ad, entries=150, sequenceid=276, filesize=12.0 K 2024-12-06T16:00:44,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/1bf26d526b95454890a854462486d30a as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/1bf26d526b95454890a854462486d30a 2024-12-06T16:00:44,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/1bf26d526b95454890a854462486d30a, entries=150, sequenceid=276, filesize=12.0 K 2024-12-06T16:00:44,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/08c8742946864c9f9d6ac5f9ea3ed45f as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/08c8742946864c9f9d6ac5f9ea3ed45f 2024-12-06T16:00:44,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/08c8742946864c9f9d6ac5f9ea3ed45f, entries=150, sequenceid=276, filesize=12.0 K 2024-12-06T16:00:44,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=107.34 KB/109920 for 81bfe42898e92490dcd2dea3186de72d in 216ms, sequenceid=276, compaction requested=false 2024-12-06T16:00:44,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:44,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:44,915 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:00:44,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81bfe42898e92490dcd2dea3186de72d 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-12-06T16:00:44,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/9bd7823797a54adf91806de1a088324c is 50, key is test_row_0/A:col10/1733500844579/Put/seqid=0 2024-12-06T16:00:44,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741894_1070 (size=14741) 2024-12-06T16:00:44,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/9bd7823797a54adf91806de1a088324c 2024-12-06T16:00:44,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:44,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500904940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:44,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:44,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500904942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:44,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:44,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:44,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:44,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500904944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:44,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:44,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/c1207aff83784fc1b6f183b098590789 is 50, key is test_row_0/B:col10/1733500844579/Put/seqid=0 2024-12-06T16:00:44,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:44,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500904945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:44,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:44,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:44,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500904947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:44,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:44,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:44,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741895_1071 (size=12301) 2024-12-06T16:00:44,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/c1207aff83784fc1b6f183b098590789 2024-12-06T16:00:44,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/56ee5c2f5fc04f40a0b753b07fffaceb is 50, key is test_row_0/C:col10/1733500844579/Put/seqid=0 2024-12-06T16:00:44,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741896_1072 (size=12301) 2024-12-06T16:00:45,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-06T16:00:45,053 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:00:45,054 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:00:45,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-06T16:00:45,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-06T16:00:45,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:45,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500905054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:45,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:45,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500905054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:45,060 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:00:45,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:45,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:45,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500905055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:45,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:45,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:45,063 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:00:45,063 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:00:45,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:45,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500905062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:45,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:45,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500905062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:45,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:45,112 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/e8a0a42aeb224ecb826258eb878db948 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/e8a0a42aeb224ecb826258eb878db948 2024-12-06T16:00:45,123 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/C of 81bfe42898e92490dcd2dea3186de72d into e8a0a42aeb224ecb826258eb878db948(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:45,124 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:45,124 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/C, priority=13, startTime=1733500844529; duration=0sec 2024-12-06T16:00:45,124 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:45,124 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:C 2024-12-06T16:00:45,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-06T16:00:45,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-06T16:00:45,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:45,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:45,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:45,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:45,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:45,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:45,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:45,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500905272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:45,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:45,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:45,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500905273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:45,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500905273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:45,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500905273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:45,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:45,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500905274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:45,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-06T16:00:45,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-06T16:00:45,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:45,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:45,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:45,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:45,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:45,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:45,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/56ee5c2f5fc04f40a0b753b07fffaceb 2024-12-06T16:00:45,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/9bd7823797a54adf91806de1a088324c as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/9bd7823797a54adf91806de1a088324c 2024-12-06T16:00:45,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/9bd7823797a54adf91806de1a088324c, entries=200, sequenceid=298, filesize=14.4 K 2024-12-06T16:00:45,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/c1207aff83784fc1b6f183b098590789 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/c1207aff83784fc1b6f183b098590789 2024-12-06T16:00:45,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/c1207aff83784fc1b6f183b098590789, entries=150, sequenceid=298, filesize=12.0 K 2024-12-06T16:00:45,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/56ee5c2f5fc04f40a0b753b07fffaceb as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/56ee5c2f5fc04f40a0b753b07fffaceb 2024-12-06T16:00:45,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/56ee5c2f5fc04f40a0b753b07fffaceb, entries=150, sequenceid=298, filesize=12.0 K 2024-12-06T16:00:45,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 81bfe42898e92490dcd2dea3186de72d in 514ms, sequenceid=298, compaction requested=true 2024-12-06T16:00:45,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:45,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:00:45,429 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:00:45,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:45,429 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:00:45,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:00:45,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:45,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:00:45,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:00:45,431 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37383 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:00:45,431 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:00:45,432 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/B is initiating minor compaction (all files) 2024-12-06T16:00:45,432 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/A is initiating minor compaction (all files) 2024-12-06T16:00:45,432 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/B in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:45,432 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/A in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:45,432 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/2ced15e61f184e47a3c714f5dcf6c11e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/1bf26d526b95454890a854462486d30a, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/c1207aff83784fc1b6f183b098590789] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=36.5 K 2024-12-06T16:00:45,432 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/1392ee1eb54541e0a91cc0899bfe8c60, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/66b953419652443984127f8ea327a8ad, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/9bd7823797a54adf91806de1a088324c] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=38.9 K 2024-12-06T16:00:45,432 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ced15e61f184e47a3c714f5dcf6c11e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733500843226 2024-12-06T16:00:45,433 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1392ee1eb54541e0a91cc0899bfe8c60, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733500843226 2024-12-06T16:00:45,433 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bf26d526b95454890a854462486d30a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733500844421 2024-12-06T16:00:45,434 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66b953419652443984127f8ea327a8ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733500844421 2024-12-06T16:00:45,434 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting c1207aff83784fc1b6f183b098590789, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733500844565 2024-12-06T16:00:45,434 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bd7823797a54adf91806de1a088324c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733500844565 2024-12-06T16:00:45,448 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#B#compaction#60 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:45,449 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/927d02c30967487bb8d95c7a75cb0ce7 is 50, key is test_row_0/B:col10/1733500844579/Put/seqid=0 2024-12-06T16:00:45,453 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#A#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:45,455 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/2cc3b666af0d42c697ce818e792c4127 is 50, key is test_row_0/A:col10/1733500844579/Put/seqid=0 2024-12-06T16:00:45,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741897_1073 (size=12983) 2024-12-06T16:00:45,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741898_1074 (size=12983) 2024-12-06T16:00:45,472 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/927d02c30967487bb8d95c7a75cb0ce7 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/927d02c30967487bb8d95c7a75cb0ce7 2024-12-06T16:00:45,481 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/B of 81bfe42898e92490dcd2dea3186de72d into 927d02c30967487bb8d95c7a75cb0ce7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:45,481 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:45,481 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/B, priority=13, startTime=1733500845429; duration=0sec 2024-12-06T16:00:45,482 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:00:45,482 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:B 2024-12-06T16:00:45,482 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:00:45,484 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37383 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:00:45,484 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/C is initiating minor compaction (all files) 2024-12-06T16:00:45,484 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/C in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:45,484 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/e8a0a42aeb224ecb826258eb878db948, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/08c8742946864c9f9d6ac5f9ea3ed45f, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/56ee5c2f5fc04f40a0b753b07fffaceb] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=36.5 K 2024-12-06T16:00:45,485 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting e8a0a42aeb224ecb826258eb878db948, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733500843226 2024-12-06T16:00:45,487 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 08c8742946864c9f9d6ac5f9ea3ed45f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733500844421 2024-12-06T16:00:45,488 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 56ee5c2f5fc04f40a0b753b07fffaceb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733500844565 2024-12-06T16:00:45,497 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#C#compaction#62 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:45,498 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/09f6ad844e984d62a2e413307fc65cf8 is 50, key is test_row_0/C:col10/1733500844579/Put/seqid=0 2024-12-06T16:00:45,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741899_1075 (size=12983) 2024-12-06T16:00:45,513 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/09f6ad844e984d62a2e413307fc65cf8 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/09f6ad844e984d62a2e413307fc65cf8 2024-12-06T16:00:45,520 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/C of 81bfe42898e92490dcd2dea3186de72d into 09f6ad844e984d62a2e413307fc65cf8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:45,520 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:45,520 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/C, priority=13, startTime=1733500845430; duration=0sec 2024-12-06T16:00:45,521 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:45,521 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:C 2024-12-06T16:00:45,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-06T16:00:45,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:45,534 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 81bfe42898e92490dcd2dea3186de72d 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-06T16:00:45,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/fea31613cc3b4efb85affccc0fe7baf6 is 50, key is test_row_0/A:col10/1733500844944/Put/seqid=0 2024-12-06T16:00:45,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741900_1076 (size=12301) 2024-12-06T16:00:45,554 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/fea31613cc3b4efb85affccc0fe7baf6 2024-12-06T16:00:45,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/d0c57682738c4f0986bad92806c85ac5 is 50, key is test_row_0/B:col10/1733500844944/Put/seqid=0 2024-12-06T16:00:45,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:45,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:45,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741901_1077 (size=12301) 2024-12-06T16:00:45,588 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/d0c57682738c4f0986bad92806c85ac5 2024-12-06T16:00:45,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/14616e15b1a34ffbb816792eab8ddcc0 is 50, key is test_row_0/C:col10/1733500844944/Put/seqid=0 2024-12-06T16:00:45,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:45,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500905627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:45,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:45,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:45,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500905629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:45,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:45,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:45,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500905631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:45,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:45,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500905632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:45,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:45,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:45,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:45,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500905635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:45,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:45,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741902_1078 (size=12301) 2024-12-06T16:00:45,646 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/14616e15b1a34ffbb816792eab8ddcc0 2024-12-06T16:00:45,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/fea31613cc3b4efb85affccc0fe7baf6 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/fea31613cc3b4efb85affccc0fe7baf6 2024-12-06T16:00:45,668 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/fea31613cc3b4efb85affccc0fe7baf6, entries=150, sequenceid=317, filesize=12.0 K 2024-12-06T16:00:45,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/d0c57682738c4f0986bad92806c85ac5 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/d0c57682738c4f0986bad92806c85ac5 2024-12-06T16:00:45,681 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/d0c57682738c4f0986bad92806c85ac5, entries=150, sequenceid=317, filesize=12.0 K 2024-12-06T16:00:45,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-06T16:00:45,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/14616e15b1a34ffbb816792eab8ddcc0 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/14616e15b1a34ffbb816792eab8ddcc0 2024-12-06T16:00:45,689 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/14616e15b1a34ffbb816792eab8ddcc0, entries=150, sequenceid=317, filesize=12.0 K 2024-12-06T16:00:45,690 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 81bfe42898e92490dcd2dea3186de72d in 157ms, sequenceid=317, compaction requested=false 2024-12-06T16:00:45,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:45,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:45,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-06T16:00:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-06T16:00:45,695 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-06T16:00:45,695 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 630 msec 2024-12-06T16:00:45,698 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 642 msec 2024-12-06T16:00:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:45,748 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:00:45,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81bfe42898e92490dcd2dea3186de72d 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-06T16:00:45,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/0e241960a7524874999038c6461e7f97 is 50, key is test_row_0/A:col10/1733500845746/Put/seqid=0 2024-12-06T16:00:45,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:45,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500905767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:45,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500905771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:45,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:45,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:45,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500905772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:45,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:45,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:45,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500905773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:45,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741903_1079 (size=12301) 2024-12-06T16:00:45,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:45,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:45,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500905774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:45,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:45,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/0e241960a7524874999038c6461e7f97 2024-12-06T16:00:45,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/28dcd6faeb3f43648fe5ea4cac4f9ed2 is 50, key is test_row_0/B:col10/1733500845746/Put/seqid=0 2024-12-06T16:00:45,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741904_1080 (size=12301) 2024-12-06T16:00:45,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/28dcd6faeb3f43648fe5ea4cac4f9ed2 2024-12-06T16:00:45,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/92104cae7974403b8b492cd0007c0c73 is 50, key is test_row_0/C:col10/1733500845746/Put/seqid=0 2024-12-06T16:00:45,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741905_1081 (size=12301) 2024-12-06T16:00:45,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/92104cae7974403b8b492cd0007c0c73 2024-12-06T16:00:45,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/0e241960a7524874999038c6461e7f97 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/0e241960a7524874999038c6461e7f97 2024-12-06T16:00:45,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/0e241960a7524874999038c6461e7f97, entries=150, sequenceid=340, filesize=12.0 K 2024-12-06T16:00:45,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/28dcd6faeb3f43648fe5ea4cac4f9ed2 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/28dcd6faeb3f43648fe5ea4cac4f9ed2 2024-12-06T16:00:45,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/28dcd6faeb3f43648fe5ea4cac4f9ed2, entries=150, sequenceid=340, filesize=12.0 K 2024-12-06T16:00:45,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/92104cae7974403b8b492cd0007c0c73 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/92104cae7974403b8b492cd0007c0c73 2024-12-06T16:00:45,881 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/2cc3b666af0d42c697ce818e792c4127 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/2cc3b666af0d42c697ce818e792c4127 2024-12-06T16:00:45,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500905882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:45,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:45,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500905882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:45,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:45,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:45,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500905882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:45,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:45,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:45,890 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/A of 81bfe42898e92490dcd2dea3186de72d into 2cc3b666af0d42c697ce818e792c4127(size=12.7 K), total size for store is 36.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:45,890 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:45,890 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/A, priority=13, startTime=1733500845429; duration=0sec 2024-12-06T16:00:45,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/92104cae7974403b8b492cd0007c0c73, entries=150, sequenceid=340, filesize=12.0 K 2024-12-06T16:00:45,890 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:45,890 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:A 2024-12-06T16:00:45,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 81bfe42898e92490dcd2dea3186de72d in 143ms, sequenceid=340, compaction requested=true 2024-12-06T16:00:45,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:45,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:00:45,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:45,892 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:00:45,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:00:45,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:00:45,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:00:45,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:00:45,892 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:00:45,893 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:00:45,893 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/B is initiating minor compaction (all files) 2024-12-06T16:00:45,893 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/B in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:45,894 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/927d02c30967487bb8d95c7a75cb0ce7, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/d0c57682738c4f0986bad92806c85ac5, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/28dcd6faeb3f43648fe5ea4cac4f9ed2] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=36.7 K 2024-12-06T16:00:45,894 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:00:45,894 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/A is initiating minor compaction (all files) 2024-12-06T16:00:45,894 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/A in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:45,894 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/2cc3b666af0d42c697ce818e792c4127, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/fea31613cc3b4efb85affccc0fe7baf6, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/0e241960a7524874999038c6461e7f97] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=36.7 K 2024-12-06T16:00:45,895 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 927d02c30967487bb8d95c7a75cb0ce7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733500844565 2024-12-06T16:00:45,895 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting d0c57682738c4f0986bad92806c85ac5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733500844942 2024-12-06T16:00:45,895 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cc3b666af0d42c697ce818e792c4127, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733500844565 2024-12-06T16:00:45,896 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 28dcd6faeb3f43648fe5ea4cac4f9ed2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733500845628 2024-12-06T16:00:45,896 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting fea31613cc3b4efb85affccc0fe7baf6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733500844942 2024-12-06T16:00:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:45,899 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:00:45,899 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81bfe42898e92490dcd2dea3186de72d 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-06T16:00:45,900 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e241960a7524874999038c6461e7f97, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733500845628 2024-12-06T16:00:45,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/7e5bf8e1a19644288d8b94a3f7b113e5 is 50, key is test_row_0/A:col10/1733500845897/Put/seqid=0 2024-12-06T16:00:45,937 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#B#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:45,938 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/19fe4af070cf43c6a66f586e0c8c1480 is 50, key is test_row_0/B:col10/1733500845746/Put/seqid=0 2024-12-06T16:00:45,954 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#A#compaction#71 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:45,955 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/d10de112330e45b4afb88afdd34db7d1 is 50, key is test_row_0/A:col10/1733500845746/Put/seqid=0 2024-12-06T16:00:45,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741906_1082 (size=14741) 2024-12-06T16:00:45,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741907_1083 (size=13085) 2024-12-06T16:00:45,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741908_1084 (size=13085) 2024-12-06T16:00:45,978 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/19fe4af070cf43c6a66f586e0c8c1480 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/19fe4af070cf43c6a66f586e0c8c1480 2024-12-06T16:00:45,990 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/B of 81bfe42898e92490dcd2dea3186de72d into 19fe4af070cf43c6a66f586e0c8c1480(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:45,990 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:45,990 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/B, priority=13, startTime=1733500845892; duration=0sec 2024-12-06T16:00:45,990 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:00:45,990 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:B 2024-12-06T16:00:45,990 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:00:45,991 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:00:45,991 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/C is initiating minor compaction (all files) 2024-12-06T16:00:45,991 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/C in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:45,992 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/09f6ad844e984d62a2e413307fc65cf8, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/14616e15b1a34ffbb816792eab8ddcc0, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/92104cae7974403b8b492cd0007c0c73] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=36.7 K 2024-12-06T16:00:45,992 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 09f6ad844e984d62a2e413307fc65cf8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733500844565 2024-12-06T16:00:45,992 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 14616e15b1a34ffbb816792eab8ddcc0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733500844942 2024-12-06T16:00:45,993 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 92104cae7974403b8b492cd0007c0c73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733500845628 2024-12-06T16:00:45,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:45,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500905987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:46,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500905989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:46,002 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#C#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:46,003 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/842b803c611e4ec5b4a0dbaf8679e63b is 50, key is test_row_0/C:col10/1733500845746/Put/seqid=0 2024-12-06T16:00:46,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741909_1085 (size=13085) 2024-12-06T16:00:46,019 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/842b803c611e4ec5b4a0dbaf8679e63b as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/842b803c611e4ec5b4a0dbaf8679e63b 2024-12-06T16:00:46,026 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/C of 81bfe42898e92490dcd2dea3186de72d into 842b803c611e4ec5b4a0dbaf8679e63b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:46,027 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:46,027 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/C, priority=13, startTime=1733500845892; duration=0sec 2024-12-06T16:00:46,027 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:46,027 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:C 2024-12-06T16:00:46,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:46,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500906092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:46,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500906092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500906092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:46,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:46,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500906102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:46,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:46,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500906112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:46,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-06T16:00:46,192 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:00:46,193 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:00:46,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-06T16:00:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-06T16:00:46,195 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:00:46,196 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:00:46,197 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:00:46,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-06T16:00:46,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:46,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500906311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:46,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:46,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:46,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500906322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:46,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:46,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-06T16:00:46,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:46,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:46,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:46,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:46,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:46,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:46,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/7e5bf8e1a19644288d8b94a3f7b113e5 2024-12-06T16:00:46,385 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/d10de112330e45b4afb88afdd34db7d1 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/d10de112330e45b4afb88afdd34db7d1 2024-12-06T16:00:46,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/293417992d714250be624998ebbc1b4c is 50, key is test_row_0/B:col10/1733500845897/Put/seqid=0 2024-12-06T16:00:46,396 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/A of 81bfe42898e92490dcd2dea3186de72d into d10de112330e45b4afb88afdd34db7d1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:46,396 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:46,396 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/A, priority=13, startTime=1733500845892; duration=0sec 2024-12-06T16:00:46,397 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:46,397 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:A 2024-12-06T16:00:46,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:46,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500906402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:46,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:46,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:46,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500906402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:46,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:46,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:46,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500906402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:46,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:46,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741910_1086 (size=12301) 2024-12-06T16:00:46,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-06T16:00:46,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:46,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:46,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:46,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:46,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:46,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-06T16:00:46,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:46,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500906622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:46,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:46,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:46,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500906632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:46,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:46,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-06T16:00:46,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:46,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:46,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:46,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:46,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:46,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:46,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/293417992d714250be624998ebbc1b4c 2024-12-06T16:00:46,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-06T16:00:46,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:46,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:46,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:46,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:46,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:46,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:46,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-06T16:00:46,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/da8d4fafaaa84b6a9fc64bfaf41779f3 is 50, key is test_row_0/C:col10/1733500845897/Put/seqid=0 2024-12-06T16:00:46,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741911_1087 (size=12301) 2024-12-06T16:00:46,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/da8d4fafaaa84b6a9fc64bfaf41779f3 2024-12-06T16:00:46,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/7e5bf8e1a19644288d8b94a3f7b113e5 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/7e5bf8e1a19644288d8b94a3f7b113e5 2024-12-06T16:00:46,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/7e5bf8e1a19644288d8b94a3f7b113e5, entries=200, sequenceid=355, filesize=14.4 K 2024-12-06T16:00:46,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/293417992d714250be624998ebbc1b4c as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/293417992d714250be624998ebbc1b4c 2024-12-06T16:00:46,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/293417992d714250be624998ebbc1b4c, entries=150, sequenceid=355, filesize=12.0 K 2024-12-06T16:00:46,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/da8d4fafaaa84b6a9fc64bfaf41779f3 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/da8d4fafaaa84b6a9fc64bfaf41779f3 2024-12-06T16:00:46,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/da8d4fafaaa84b6a9fc64bfaf41779f3, entries=150, sequenceid=355, filesize=12.0 K 2024-12-06T16:00:46,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 81bfe42898e92490dcd2dea3186de72d in 973ms, sequenceid=355, compaction requested=false 2024-12-06T16:00:46,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:46,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:46,918 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:00:46,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81bfe42898e92490dcd2dea3186de72d 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-06T16:00:46,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/9d7fe631002f4d0c8b1059c5c5046322 is 50, key is test_row_0/A:col10/1733500845984/Put/seqid=0 2024-12-06T16:00:46,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:46,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500906952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:46,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:46,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:46,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500906954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:46,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:46,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500906954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:46,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:46,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:46,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:46,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741912_1088 (size=17181) 2024-12-06T16:00:46,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-06T16:00:46,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:46,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:46,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:46,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:46,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:46,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:47,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:47,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:47,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500907062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:47,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500907062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:47,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:47,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500907071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:47,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:47,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:47,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:47,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-06T16:00:47,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:47,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:47,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:47,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:47,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:47,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:47,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:47,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500907132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:47,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:47,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:47,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500907141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:47,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:47,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:47,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:47,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:47,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500907272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:47,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:47,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:47,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:47,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:47,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500907272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:47,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:47,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:47,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:47,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-06T16:00:47,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:47,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:47,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:47,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:47,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:47,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:47,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:47,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500907282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:47,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:47,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:47,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:47,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-06T16:00:47,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/9d7fe631002f4d0c8b1059c5c5046322 2024-12-06T16:00:47,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/6a7017dd29d148d994eebf69b88e4538 is 50, key is test_row_0/B:col10/1733500845984/Put/seqid=0 2024-12-06T16:00:47,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741913_1089 (size=12301) 2024-12-06T16:00:47,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/6a7017dd29d148d994eebf69b88e4538 2024-12-06T16:00:47,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/1c2c85f9127b4ecb95ceec2e599b6f41 is 50, key is test_row_0/C:col10/1733500845984/Put/seqid=0 2024-12-06T16:00:47,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741914_1090 (size=12301) 2024-12-06T16:00:47,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-06T16:00:47,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:47,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:47,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:47,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:47,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:47,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:47,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500907581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:47,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500907582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:47,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:47,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:47,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:47,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:47,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:47,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:47,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-06T16:00:47,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:47,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:47,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:47,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:47,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:47,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500907592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:47,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:47,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:47,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:47,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-06T16:00:47,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:47,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:47,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:47,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:47,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:47,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/1c2c85f9127b4ecb95ceec2e599b6f41 2024-12-06T16:00:47,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/9d7fe631002f4d0c8b1059c5c5046322 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/9d7fe631002f4d0c8b1059c5c5046322 2024-12-06T16:00:47,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/9d7fe631002f4d0c8b1059c5c5046322, entries=250, sequenceid=382, filesize=16.8 K 2024-12-06T16:00:47,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/6a7017dd29d148d994eebf69b88e4538 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/6a7017dd29d148d994eebf69b88e4538 2024-12-06T16:00:47,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/6a7017dd29d148d994eebf69b88e4538, entries=150, sequenceid=382, filesize=12.0 K 2024-12-06T16:00:47,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/1c2c85f9127b4ecb95ceec2e599b6f41 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/1c2c85f9127b4ecb95ceec2e599b6f41 2024-12-06T16:00:47,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/1c2c85f9127b4ecb95ceec2e599b6f41, entries=150, sequenceid=382, filesize=12.0 K 2024-12-06T16:00:47,862 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 81bfe42898e92490dcd2dea3186de72d in 944ms, sequenceid=382, compaction requested=true 2024-12-06T16:00:47,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:47,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:00:47,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:47,862 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:00:47,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:00:47,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:00:47,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:00:47,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-06T16:00:47,863 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:00:47,864 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45007 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:00:47,864 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/A is initiating minor compaction (all files) 2024-12-06T16:00:47,864 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/A in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:47,864 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/d10de112330e45b4afb88afdd34db7d1, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/7e5bf8e1a19644288d8b94a3f7b113e5, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/9d7fe631002f4d0c8b1059c5c5046322] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=44.0 K 2024-12-06T16:00:47,866 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:00:47,866 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/B is initiating minor compaction (all files) 2024-12-06T16:00:47,866 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/B in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:47,866 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/19fe4af070cf43c6a66f586e0c8c1480, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/293417992d714250be624998ebbc1b4c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/6a7017dd29d148d994eebf69b88e4538] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=36.8 K 2024-12-06T16:00:47,866 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting d10de112330e45b4afb88afdd34db7d1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733500845628 2024-12-06T16:00:47,867 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 19fe4af070cf43c6a66f586e0c8c1480, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733500845628 2024-12-06T16:00:47,867 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e5bf8e1a19644288d8b94a3f7b113e5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733500845766 2024-12-06T16:00:47,868 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 293417992d714250be624998ebbc1b4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733500845766 2024-12-06T16:00:47,868 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d7fe631002f4d0c8b1059c5c5046322, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733500845981 2024-12-06T16:00:47,868 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a7017dd29d148d994eebf69b88e4538, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733500845984 2024-12-06T16:00:47,882 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#A#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:47,885 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/9ce1abc1b8b643668155df102f8ba56c is 50, key is test_row_0/A:col10/1733500845984/Put/seqid=0 2024-12-06T16:00:47,886 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#B#compaction#79 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:47,888 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/1f201f9f69c645e2be1ec2a0dbe2b645 is 50, key is test_row_0/B:col10/1733500845984/Put/seqid=0 2024-12-06T16:00:47,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741915_1091 (size=13187) 2024-12-06T16:00:47,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741916_1092 (size=13187) 2024-12-06T16:00:47,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-06T16:00:47,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:47,901 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 81bfe42898e92490dcd2dea3186de72d 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-06T16:00:47,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/42ed87278f284c75a13b7a971388f366 is 50, key is test_row_0/A:col10/1733500846943/Put/seqid=0 2024-12-06T16:00:47,908 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/9ce1abc1b8b643668155df102f8ba56c as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/9ce1abc1b8b643668155df102f8ba56c 2024-12-06T16:00:47,918 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/A of 81bfe42898e92490dcd2dea3186de72d into 9ce1abc1b8b643668155df102f8ba56c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:47,918 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:47,918 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/A, priority=13, startTime=1733500847862; duration=0sec 2024-12-06T16:00:47,918 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:00:47,918 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:A 2024-12-06T16:00:47,918 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:00:47,921 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:00:47,921 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/C is initiating minor compaction (all files) 2024-12-06T16:00:47,921 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/C in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:47,921 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/842b803c611e4ec5b4a0dbaf8679e63b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/da8d4fafaaa84b6a9fc64bfaf41779f3, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/1c2c85f9127b4ecb95ceec2e599b6f41] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=36.8 K 2024-12-06T16:00:47,922 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 842b803c611e4ec5b4a0dbaf8679e63b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733500845628 2024-12-06T16:00:47,923 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting da8d4fafaaa84b6a9fc64bfaf41779f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733500845766 2024-12-06T16:00:47,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741917_1093 (size=12301) 2024-12-06T16:00:47,924 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c2c85f9127b4ecb95ceec2e599b6f41, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733500845984 2024-12-06T16:00:47,925 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/42ed87278f284c75a13b7a971388f366 2024-12-06T16:00:47,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/2ced8238925844049b539f8441fd4bc8 is 50, key is test_row_0/B:col10/1733500846943/Put/seqid=0 2024-12-06T16:00:47,949 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#C#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:47,950 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/bd2c3b8609fc4af689ac6866265fa3d4 is 50, key is test_row_0/C:col10/1733500845984/Put/seqid=0 2024-12-06T16:00:47,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741918_1094 (size=12301) 2024-12-06T16:00:47,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741919_1095 (size=13187) 2024-12-06T16:00:47,987 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/bd2c3b8609fc4af689ac6866265fa3d4 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/bd2c3b8609fc4af689ac6866265fa3d4 2024-12-06T16:00:48,015 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/C of 81bfe42898e92490dcd2dea3186de72d into bd2c3b8609fc4af689ac6866265fa3d4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:48,015 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:48,015 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/C, priority=13, startTime=1733500847862; duration=0sec 2024-12-06T16:00:48,015 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:48,015 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:C 2024-12-06T16:00:48,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:48,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:48,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:48,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500908140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:48,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:48,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500908142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:48,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:48,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:48,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:48,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500908142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:48,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:48,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:48,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500908151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:48,161 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 85bef17d9292,46105,1733500827732 for 'test_row_2' in 81bfe42898e92490dcd2dea3186de72d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T16:00:48,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:48,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:48,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500908162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:48,167 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 85bef17d9292,46105,1733500827732 for 'test_row_2' in 81bfe42898e92490dcd2dea3186de72d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T16:00:48,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:48,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:48,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500908252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:48,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500908252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:48,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500908252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:48,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:48,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:48,309 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/1f201f9f69c645e2be1ec2a0dbe2b645 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/1f201f9f69c645e2be1ec2a0dbe2b645 2024-12-06T16:00:48,322 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/B of 81bfe42898e92490dcd2dea3186de72d into 1f201f9f69c645e2be1ec2a0dbe2b645(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:48,322 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:48,322 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/B, priority=13, startTime=1733500847862; duration=0sec 2024-12-06T16:00:48,322 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:48,322 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:B 2024-12-06T16:00:48,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-06T16:00:48,357 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/2ced8238925844049b539f8441fd4bc8 2024-12-06T16:00:48,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/42fe32e9d0024881bafcabf4d91f8e0b is 50, key is test_row_0/C:col10/1733500846943/Put/seqid=0 2024-12-06T16:00:48,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741920_1096 (size=12301) 2024-12-06T16:00:48,377 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/42fe32e9d0024881bafcabf4d91f8e0b 2024-12-06T16:00:48,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/42ed87278f284c75a13b7a971388f366 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/42ed87278f284c75a13b7a971388f366 2024-12-06T16:00:48,394 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/42ed87278f284c75a13b7a971388f366, entries=150, sequenceid=395, filesize=12.0 K 2024-12-06T16:00:48,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/2ced8238925844049b539f8441fd4bc8 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/2ced8238925844049b539f8441fd4bc8 2024-12-06T16:00:48,406 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/2ced8238925844049b539f8441fd4bc8, entries=150, sequenceid=395, filesize=12.0 K 2024-12-06T16:00:48,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/42fe32e9d0024881bafcabf4d91f8e0b as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/42fe32e9d0024881bafcabf4d91f8e0b 2024-12-06T16:00:48,419 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/42fe32e9d0024881bafcabf4d91f8e0b, entries=150, sequenceid=395, filesize=12.0 K 2024-12-06T16:00:48,424 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 81bfe42898e92490dcd2dea3186de72d in 523ms, sequenceid=395, compaction requested=false 2024-12-06T16:00:48,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:48,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:48,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-06T16:00:48,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-06T16:00:48,430 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-06T16:00:48,430 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2300 sec 2024-12-06T16:00:48,433 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.2380 sec 2024-12-06T16:00:48,472 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:00:48,472 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81bfe42898e92490dcd2dea3186de72d 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-06T16:00:48,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:48,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/692c0170d2244846a58a055ac511b6d7 is 50, key is test_row_0/A:col10/1733500848468/Put/seqid=0 2024-12-06T16:00:48,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:48,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500908493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:48,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:48,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500908494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:48,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:48,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:48,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:48,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500908496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:48,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:48,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741921_1097 (size=14741) 2024-12-06T16:00:48,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/692c0170d2244846a58a055ac511b6d7 2024-12-06T16:00:48,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/1fa2c75a8fea4157b93ab23a13655489 is 50, key is test_row_0/B:col10/1733500848468/Put/seqid=0 2024-12-06T16:00:48,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741922_1098 (size=12301) 2024-12-06T16:00:48,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:48,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500908601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:48,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:48,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:48,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500908602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:48,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:48,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500908602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:48,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:48,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:48,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:48,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500908812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:48,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:48,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:48,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500908813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:48,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:48,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500908814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:48,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:48,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:48,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:48,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/1fa2c75a8fea4157b93ab23a13655489 2024-12-06T16:00:48,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/022e94ec72014382a1d62f48bba40b03 is 50, key is test_row_0/C:col10/1733500848468/Put/seqid=0 2024-12-06T16:00:49,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741923_1099 (size=12301) 2024-12-06T16:00:49,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/022e94ec72014382a1d62f48bba40b03 2024-12-06T16:00:49,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/692c0170d2244846a58a055ac511b6d7 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/692c0170d2244846a58a055ac511b6d7 2024-12-06T16:00:49,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/692c0170d2244846a58a055ac511b6d7, entries=200, sequenceid=422, filesize=14.4 K 2024-12-06T16:00:49,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/1fa2c75a8fea4157b93ab23a13655489 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/1fa2c75a8fea4157b93ab23a13655489 2024-12-06T16:00:49,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/1fa2c75a8fea4157b93ab23a13655489, entries=150, sequenceid=422, filesize=12.0 K 2024-12-06T16:00:49,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/022e94ec72014382a1d62f48bba40b03 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/022e94ec72014382a1d62f48bba40b03 2024-12-06T16:00:49,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/022e94ec72014382a1d62f48bba40b03, entries=150, sequenceid=422, filesize=12.0 K 2024-12-06T16:00:49,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 81bfe42898e92490dcd2dea3186de72d in 586ms, sequenceid=422, compaction requested=true 2024-12-06T16:00:49,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:49,058 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:00:49,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:00:49,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:49,059 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:00:49,060 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:00:49,060 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/A is initiating minor compaction (all files) 2024-12-06T16:00:49,060 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/A in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:49,061 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/9ce1abc1b8b643668155df102f8ba56c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/42ed87278f284c75a13b7a971388f366, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/692c0170d2244846a58a055ac511b6d7] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=39.3 K 2024-12-06T16:00:49,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:00:49,061 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:00:49,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:49,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:00:49,061 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ce1abc1b8b643668155df102f8ba56c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733500845984 2024-12-06T16:00:49,061 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/B is initiating minor compaction (all files) 2024-12-06T16:00:49,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:00:49,061 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/B in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:49,061 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/1f201f9f69c645e2be1ec2a0dbe2b645, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/2ced8238925844049b539f8441fd4bc8, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/1fa2c75a8fea4157b93ab23a13655489] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=36.9 K 2024-12-06T16:00:49,062 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42ed87278f284c75a13b7a971388f366, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733500846943 2024-12-06T16:00:49,062 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f201f9f69c645e2be1ec2a0dbe2b645, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733500845984 2024-12-06T16:00:49,063 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 692c0170d2244846a58a055ac511b6d7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1733500848126 2024-12-06T16:00:49,065 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ced8238925844049b539f8441fd4bc8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733500846943 2024-12-06T16:00:49,065 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fa2c75a8fea4157b93ab23a13655489, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1733500848126 2024-12-06T16:00:49,086 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#A#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:49,087 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/5c0caf8dd8334fb5bcf03bb3a3978cd5 is 50, key is test_row_0/A:col10/1733500848468/Put/seqid=0 2024-12-06T16:00:49,091 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#B#compaction#88 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:49,092 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/ff25f05bc08b4512b2c6d92c6fbe8836 is 50, key is test_row_0/B:col10/1733500848468/Put/seqid=0 2024-12-06T16:00:49,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741925_1101 (size=13289) 2024-12-06T16:00:49,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741924_1100 (size=13289) 2024-12-06T16:00:49,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:49,124 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:00:49,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81bfe42898e92490dcd2dea3186de72d 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-06T16:00:49,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/818e4bedca64434cb97f63e35dbfa70e is 50, key is test_row_0/A:col10/1733500848492/Put/seqid=0 2024-12-06T16:00:49,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741926_1102 (size=17181) 2024-12-06T16:00:49,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/818e4bedca64434cb97f63e35dbfa70e 2024-12-06T16:00:49,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/5c1b93dd729d4588bf8c5399d6ac776e is 50, key is test_row_0/B:col10/1733500848492/Put/seqid=0 2024-12-06T16:00:49,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741927_1103 (size=12301) 2024-12-06T16:00:49,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:49,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500909229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:49,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:49,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500909231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:49,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:49,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500909234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:49,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:49,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:49,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:49,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:49,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:49,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:49,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:49,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500909342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:49,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:49,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:49,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:49,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:49,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500909343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:49,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:49,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500909351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:49,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:49,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:49,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:49,533 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/ff25f05bc08b4512b2c6d92c6fbe8836 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/ff25f05bc08b4512b2c6d92c6fbe8836 2024-12-06T16:00:49,535 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/5c0caf8dd8334fb5bcf03bb3a3978cd5 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/5c0caf8dd8334fb5bcf03bb3a3978cd5 2024-12-06T16:00:49,553 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/B of 81bfe42898e92490dcd2dea3186de72d into ff25f05bc08b4512b2c6d92c6fbe8836(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:49,553 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/A of 81bfe42898e92490dcd2dea3186de72d into 5c0caf8dd8334fb5bcf03bb3a3978cd5(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:49,553 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:49,553 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:49,553 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/B, priority=13, startTime=1733500849059; duration=0sec 2024-12-06T16:00:49,553 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/A, priority=13, startTime=1733500849058; duration=0sec 2024-12-06T16:00:49,553 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:00:49,553 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:B 2024-12-06T16:00:49,554 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:00:49,554 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:49,554 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:A 2024-12-06T16:00:49,555 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:00:49,555 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/C is initiating minor compaction (all files) 2024-12-06T16:00:49,556 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/C in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:49,556 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/bd2c3b8609fc4af689ac6866265fa3d4, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/42fe32e9d0024881bafcabf4d91f8e0b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/022e94ec72014382a1d62f48bba40b03] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=36.9 K 2024-12-06T16:00:49,557 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting bd2c3b8609fc4af689ac6866265fa3d4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733500845984 2024-12-06T16:00:49,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500909552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:49,557 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 42fe32e9d0024881bafcabf4d91f8e0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733500846943 2024-12-06T16:00:49,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:49,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:49,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:49,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:49,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500909553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:49,558 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 022e94ec72014382a1d62f48bba40b03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1733500848126 2024-12-06T16:00:49,558 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:49,558 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:49,558 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:49,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:49,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500909562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:49,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:49,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:49,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:49,569 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#C#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:49,570 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/1d63aadd7d4743048773b6820ac28d5a is 50, key is test_row_0/C:col10/1733500848468/Put/seqid=0 2024-12-06T16:00:49,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741928_1104 (size=13289) 2024-12-06T16:00:49,604 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/1d63aadd7d4743048773b6820ac28d5a as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/1d63aadd7d4743048773b6820ac28d5a 2024-12-06T16:00:49,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/5c1b93dd729d4588bf8c5399d6ac776e 2024-12-06T16:00:49,616 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/C of 81bfe42898e92490dcd2dea3186de72d into 1d63aadd7d4743048773b6820ac28d5a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:49,616 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:49,616 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/C, priority=13, startTime=1733500849061; duration=0sec 2024-12-06T16:00:49,616 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:49,616 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:C 2024-12-06T16:00:49,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/f4b0f41751164b72add566aae08bb760 is 50, key is test_row_0/C:col10/1733500848492/Put/seqid=0 2024-12-06T16:00:49,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741929_1105 (size=12301) 2024-12-06T16:00:49,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/f4b0f41751164b72add566aae08bb760 2024-12-06T16:00:49,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/818e4bedca64434cb97f63e35dbfa70e as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/818e4bedca64434cb97f63e35dbfa70e 2024-12-06T16:00:49,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/818e4bedca64434cb97f63e35dbfa70e, entries=250, sequenceid=436, filesize=16.8 K 2024-12-06T16:00:49,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/5c1b93dd729d4588bf8c5399d6ac776e as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/5c1b93dd729d4588bf8c5399d6ac776e 2024-12-06T16:00:49,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/5c1b93dd729d4588bf8c5399d6ac776e, entries=150, sequenceid=436, filesize=12.0 K 2024-12-06T16:00:49,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/f4b0f41751164b72add566aae08bb760 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/f4b0f41751164b72add566aae08bb760 2024-12-06T16:00:49,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/f4b0f41751164b72add566aae08bb760, entries=150, sequenceid=436, filesize=12.0 K 2024-12-06T16:00:49,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 81bfe42898e92490dcd2dea3186de72d in 558ms, sequenceid=436, compaction requested=false 2024-12-06T16:00:49,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:49,865 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:00:49,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81bfe42898e92490dcd2dea3186de72d 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-06T16:00:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:49,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/1f7986ab167f4b1fad9952be1f689e3e is 50, key is test_row_0/A:col10/1733500849863/Put/seqid=0 2024-12-06T16:00:49,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741930_1106 (size=14741) 2024-12-06T16:00:49,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:49,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500909894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:49,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:49,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:49,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500909895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:49,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:49,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:49,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:49,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:49,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:49,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:49,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500909898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:49,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:49,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:49,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:50,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500910001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:50,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:50,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:50,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500910002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:50,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:50,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:50,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500910012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:50,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:50,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:50,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500910172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:50,175 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 85bef17d9292,46105,1733500827732 for 'test_row_2' in 81bfe42898e92490dcd2dea3186de72d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T16:00:50,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:50,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:50,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500910192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:50,196 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 85bef17d9292,46105,1733500827732 for 'test_row_2' in 81bfe42898e92490dcd2dea3186de72d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T16:00:50,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:50,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:50,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500910212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:50,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:50,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:50,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500910212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:50,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:50,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:50,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500910221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:50,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:50,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/1f7986ab167f4b1fad9952be1f689e3e 2024-12-06T16:00:50,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/c9dd52db4bef4139ac7e481feee7150b is 50, key is test_row_0/B:col10/1733500849863/Put/seqid=0 2024-12-06T16:00:50,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741931_1107 (size=12301) 2024-12-06T16:00:50,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/c9dd52db4bef4139ac7e481feee7150b 2024-12-06T16:00:50,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/e56c148a815f42cdaa8d34e7beed587a is 50, key is test_row_0/C:col10/1733500849863/Put/seqid=0 2024-12-06T16:00:50,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741932_1108 (size=12301) 2024-12-06T16:00:50,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/e56c148a815f42cdaa8d34e7beed587a 2024-12-06T16:00:50,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-06T16:00:50,363 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:00:50,364 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:00:50,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-06T16:00:50,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-06T16:00:50,370 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:00:50,371 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:00:50,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:00:50,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/1f7986ab167f4b1fad9952be1f689e3e as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/1f7986ab167f4b1fad9952be1f689e3e 2024-12-06T16:00:50,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/1f7986ab167f4b1fad9952be1f689e3e, entries=200, sequenceid=462, filesize=14.4 K 2024-12-06T16:00:50,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/c9dd52db4bef4139ac7e481feee7150b as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/c9dd52db4bef4139ac7e481feee7150b 2024-12-06T16:00:50,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/c9dd52db4bef4139ac7e481feee7150b, entries=150, sequenceid=462, filesize=12.0 K 2024-12-06T16:00:50,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/e56c148a815f42cdaa8d34e7beed587a as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/e56c148a815f42cdaa8d34e7beed587a 2024-12-06T16:00:50,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/e56c148a815f42cdaa8d34e7beed587a, entries=150, sequenceid=462, filesize=12.0 K 2024-12-06T16:00:50,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 81bfe42898e92490dcd2dea3186de72d in 544ms, sequenceid=462, compaction requested=true 2024-12-06T16:00:50,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:50,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:00:50,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:50,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:00:50,410 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:00:50,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:00:50,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:00:50,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-06T16:00:50,411 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:00:50,412 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45211 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:00:50,412 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/A is initiating minor compaction (all files) 2024-12-06T16:00:50,413 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/A in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:50,413 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/5c0caf8dd8334fb5bcf03bb3a3978cd5, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/818e4bedca64434cb97f63e35dbfa70e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/1f7986ab167f4b1fad9952be1f689e3e] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=44.2 K 2024-12-06T16:00:50,413 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:00:50,413 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/B is initiating minor compaction (all files) 2024-12-06T16:00:50,413 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/B in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:50,413 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/ff25f05bc08b4512b2c6d92c6fbe8836, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/5c1b93dd729d4588bf8c5399d6ac776e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/c9dd52db4bef4139ac7e481feee7150b] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=37.0 K 2024-12-06T16:00:50,413 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c0caf8dd8334fb5bcf03bb3a3978cd5, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1733500848126 2024-12-06T16:00:50,414 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting ff25f05bc08b4512b2c6d92c6fbe8836, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1733500848126 2024-12-06T16:00:50,414 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 818e4bedca64434cb97f63e35dbfa70e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1733500848490 2024-12-06T16:00:50,418 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c1b93dd729d4588bf8c5399d6ac776e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1733500848492 2024-12-06T16:00:50,418 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f7986ab167f4b1fad9952be1f689e3e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733500849162 2024-12-06T16:00:50,419 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting c9dd52db4bef4139ac7e481feee7150b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733500849162 2024-12-06T16:00:50,440 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#A#compaction#96 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:50,441 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/bb63e022a32843fe805a2fbf8883c106 is 50, key is test_row_0/A:col10/1733500849863/Put/seqid=0 2024-12-06T16:00:50,450 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#B#compaction#97 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:50,451 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/da1f1c647c004424ae51e876737812c5 is 50, key is test_row_0/B:col10/1733500849863/Put/seqid=0 2024-12-06T16:00:50,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-06T16:00:50,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741933_1109 (size=13391) 2024-12-06T16:00:50,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741934_1110 (size=13391) 2024-12-06T16:00:50,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:50,528 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:00:50,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81bfe42898e92490dcd2dea3186de72d 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-06T16:00:50,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-06T16:00:50,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:50,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:50,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:50,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:50,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:50,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:50,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/87cdfc890b9c49e495ed4c3fe7dfbefb is 50, key is test_row_0/A:col10/1733500849889/Put/seqid=0 2024-12-06T16:00:50,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741935_1111 (size=12301) 2024-12-06T16:00:50,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:50,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500910573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:50,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:50,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500910573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:50,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:50,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:50,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:50,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500910573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:50,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:50,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:50,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-06T16:00:50,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500910682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:50,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:50,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:50,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:50,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:50,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:50,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:50,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:50,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:50,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:50,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500910682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:50,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500910682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:50,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-06T16:00:50,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:50,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:50,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-06T16:00:50,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:50,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:50,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:50,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:50,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:50,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:50,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:50,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500910904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:50,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500910904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:50,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:50,919 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/bb63e022a32843fe805a2fbf8883c106 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/bb63e022a32843fe805a2fbf8883c106 2024-12-06T16:00:50,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:50,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500910904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:50,931 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/da1f1c647c004424ae51e876737812c5 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/da1f1c647c004424ae51e876737812c5 2024-12-06T16:00:50,945 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/A of 81bfe42898e92490dcd2dea3186de72d into bb63e022a32843fe805a2fbf8883c106(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:50,945 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:50,945 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/A, priority=13, startTime=1733500850410; duration=0sec 2024-12-06T16:00:50,945 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:00:50,945 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:A 2024-12-06T16:00:50,952 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/B of 81bfe42898e92490dcd2dea3186de72d into da1f1c647c004424ae51e876737812c5(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:50,952 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:00:50,952 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:50,952 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/B, priority=13, startTime=1733500850410; duration=0sec 2024-12-06T16:00:50,954 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:50,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/87cdfc890b9c49e495ed4c3fe7dfbefb 2024-12-06T16:00:50,954 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:B 2024-12-06T16:00:50,955 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:00:50,955 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/C is initiating minor compaction (all files) 2024-12-06T16:00:50,955 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/C in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:50,955 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/1d63aadd7d4743048773b6820ac28d5a, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/f4b0f41751164b72add566aae08bb760, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/e56c148a815f42cdaa8d34e7beed587a] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=37.0 K 2024-12-06T16:00:50,956 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d63aadd7d4743048773b6820ac28d5a, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1733500848126 2024-12-06T16:00:50,958 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4b0f41751164b72add566aae08bb760, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1733500848492 2024-12-06T16:00:50,959 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting e56c148a815f42cdaa8d34e7beed587a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733500849162 2024-12-06T16:00:50,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/74dc70a38f474570bffac688ae22d836 is 50, key is test_row_0/B:col10/1733500849889/Put/seqid=0 2024-12-06T16:00:50,982 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#C#compaction#100 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:50,983 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/a40c7e21c99d4b02aa7ae4e667d38cc7 is 50, key is test_row_0/C:col10/1733500849863/Put/seqid=0 2024-12-06T16:00:51,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-06T16:00:51,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-06T16:00:51,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:51,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:51,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:51,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:51,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:51,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:51,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741937_1113 (size=13391) 2024-12-06T16:00:51,051 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/a40c7e21c99d4b02aa7ae4e667d38cc7 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/a40c7e21c99d4b02aa7ae4e667d38cc7 2024-12-06T16:00:51,061 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/C of 81bfe42898e92490dcd2dea3186de72d into a40c7e21c99d4b02aa7ae4e667d38cc7(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:51,061 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:51,061 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/C, priority=13, startTime=1733500850411; duration=0sec 2024-12-06T16:00:51,061 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:51,061 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:C 2024-12-06T16:00:51,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741936_1112 (size=12301) 2024-12-06T16:00:51,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-06T16:00:51,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:51,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:51,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:51,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:51,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:51,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500911222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:51,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:51,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:51,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:51,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500911222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:51,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:51,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:51,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:51,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500911232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:51,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:51,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:51,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:51,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-06T16:00:51,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:51,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:51,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:51,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:51,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:51,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/74dc70a38f474570bffac688ae22d836 2024-12-06T16:00:51,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-06T16:00:51,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:51,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:51,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:51,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:51,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:00:51,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/c7d2168c0493421eb85012fdd10c1af3 is 50, key is test_row_0/C:col10/1733500849889/Put/seqid=0 2024-12-06T16:00:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-06T16:00:51,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741938_1114 (size=12301) 2024-12-06T16:00:51,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/c7d2168c0493421eb85012fdd10c1af3 2024-12-06T16:00:51,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/87cdfc890b9c49e495ed4c3fe7dfbefb as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/87cdfc890b9c49e495ed4c3fe7dfbefb 2024-12-06T16:00:51,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/87cdfc890b9c49e495ed4c3fe7dfbefb, entries=150, sequenceid=476, filesize=12.0 K 2024-12-06T16:00:51,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/74dc70a38f474570bffac688ae22d836 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/74dc70a38f474570bffac688ae22d836 2024-12-06T16:00:51,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/74dc70a38f474570bffac688ae22d836, entries=150, sequenceid=476, filesize=12.0 K 2024-12-06T16:00:51,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/c7d2168c0493421eb85012fdd10c1af3 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/c7d2168c0493421eb85012fdd10c1af3 2024-12-06T16:00:51,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/c7d2168c0493421eb85012fdd10c1af3, entries=150, sequenceid=476, filesize=12.0 K 2024-12-06T16:00:51,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 81bfe42898e92490dcd2dea3186de72d in 1080ms, sequenceid=476, compaction requested=false 2024-12-06T16:00:51,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:51,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-06T16:00:51,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:51,628 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 81bfe42898e92490dcd2dea3186de72d 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-06T16:00:51,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/c271b18083e242de858228aeb018595b is 50, key is test_row_0/A:col10/1733500850571/Put/seqid=0 2024-12-06T16:00:51,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741939_1115 (size=12301) 2024-12-06T16:00:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:51,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. as already flushing 2024-12-06T16:00:51,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:51,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500911793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:51,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:51,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:51,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:51,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:51,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500911794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:51,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:51,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:51,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:51,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:51,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500911795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:51,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:51,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:51,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:51,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:51,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500911902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:51,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:51,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:51,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:51,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:51,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500911912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:51,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:51,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:51,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:51,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:51,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500911912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:51,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:51,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:51,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:52,084 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/c271b18083e242de858228aeb018595b 2024-12-06T16:00:52,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/e1523ea29bbd4072ac809a383193e80d is 50, key is test_row_0/B:col10/1733500850571/Put/seqid=0 2024-12-06T16:00:52,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:52,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500912113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:52,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:52,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:52,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:52,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:52,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500912122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:52,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:52,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:52,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:52,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:52,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500912122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:52,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741940_1116 (size=12301) 2024-12-06T16:00:52,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:52,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500912432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:52,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500912432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:52,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:52,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:52,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:52,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500912432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:52,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:52,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:52,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:52,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:52,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:52,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-06T16:00:52,535 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/e1523ea29bbd4072ac809a383193e80d 2024-12-06T16:00:52,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/6cb91ab09bc84f228e839b9dba15f645 is 50, key is test_row_0/C:col10/1733500850571/Put/seqid=0 2024-12-06T16:00:52,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741941_1117 (size=12301) 2024-12-06T16:00:52,554 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/6cb91ab09bc84f228e839b9dba15f645 2024-12-06T16:00:52,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/c271b18083e242de858228aeb018595b as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/c271b18083e242de858228aeb018595b 2024-12-06T16:00:52,572 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/c271b18083e242de858228aeb018595b, entries=150, sequenceid=501, filesize=12.0 K 2024-12-06T16:00:52,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/e1523ea29bbd4072ac809a383193e80d as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/e1523ea29bbd4072ac809a383193e80d 2024-12-06T16:00:52,579 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/e1523ea29bbd4072ac809a383193e80d, entries=150, sequenceid=501, filesize=12.0 K 2024-12-06T16:00:52,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/6cb91ab09bc84f228e839b9dba15f645 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/6cb91ab09bc84f228e839b9dba15f645 2024-12-06T16:00:52,589 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/6cb91ab09bc84f228e839b9dba15f645, entries=150, sequenceid=501, filesize=12.0 K 2024-12-06T16:00:52,590 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 81bfe42898e92490dcd2dea3186de72d in 962ms, sequenceid=501, compaction requested=true 2024-12-06T16:00:52,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:52,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:52,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-06T16:00:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-06T16:00:52,596 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-06T16:00:52,596 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2210 sec 2024-12-06T16:00:52,600 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.2340 sec 2024-12-06T16:00:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:52,945 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:00:52,946 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81bfe42898e92490dcd2dea3186de72d 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-06T16:00:52,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/5b7c2b55931946a48a219bfa0a3aea80 is 50, key is test_row_0/A:col10/1733500852946/Put/seqid=0 2024-12-06T16:00:52,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741942_1118 (size=12301) 2024-12-06T16:00:52,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/5b7c2b55931946a48a219bfa0a3aea80 2024-12-06T16:00:53,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:53,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500913003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:53,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:53,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:53,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:53,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500913007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:53,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:53,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:53,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:53,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:53,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:53,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/34d521cd0dc2468db37d03f31224da2e is 50, key is test_row_0/B:col10/1733500852946/Put/seqid=0 2024-12-06T16:00:53,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500913010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:53,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:53,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:53,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:53,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741943_1119 (size=12301) 2024-12-06T16:00:53,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:53,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500913122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:53,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:53,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:53,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:53,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500913122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:53,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:53,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:53,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:53,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:53,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:53,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500913132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:53,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:53,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:53,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:53,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:53,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:53,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500913332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:53,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500913331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:53,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:53,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500913342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:53,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:53,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:53,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:53,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/34d521cd0dc2468db37d03f31224da2e 2024-12-06T16:00:53,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/7b9600ffe7e24320bd4ecfb313efed87 is 50, key is test_row_0/C:col10/1733500852946/Put/seqid=0 2024-12-06T16:00:53,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741944_1120 (size=12301) 2024-12-06T16:00:53,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/7b9600ffe7e24320bd4ecfb313efed87 2024-12-06T16:00:53,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/5b7c2b55931946a48a219bfa0a3aea80 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/5b7c2b55931946a48a219bfa0a3aea80 2024-12-06T16:00:53,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/5b7c2b55931946a48a219bfa0a3aea80, entries=150, sequenceid=517, filesize=12.0 K 2024-12-06T16:00:53,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/34d521cd0dc2468db37d03f31224da2e as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/34d521cd0dc2468db37d03f31224da2e 2024-12-06T16:00:53,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/34d521cd0dc2468db37d03f31224da2e, entries=150, sequenceid=517, filesize=12.0 K 2024-12-06T16:00:53,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/7b9600ffe7e24320bd4ecfb313efed87 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/7b9600ffe7e24320bd4ecfb313efed87 2024-12-06T16:00:53,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/7b9600ffe7e24320bd4ecfb313efed87, entries=150, sequenceid=517, filesize=12.0 K 2024-12-06T16:00:53,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 81bfe42898e92490dcd2dea3186de72d in 558ms, sequenceid=517, compaction requested=true 2024-12-06T16:00:53,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:53,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:00:53,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:53,504 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T16:00:53,504 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T16:00:53,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:00:53,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:53,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:00:53,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:00:53,507 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50294 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T16:00:53,507 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/B is initiating minor compaction (all files) 2024-12-06T16:00:53,507 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/B in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:53,507 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/da1f1c647c004424ae51e876737812c5, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/74dc70a38f474570bffac688ae22d836, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/e1523ea29bbd4072ac809a383193e80d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/34d521cd0dc2468db37d03f31224da2e] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=49.1 K 2024-12-06T16:00:53,508 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50294 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T16:00:53,508 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/A is initiating minor compaction (all files) 2024-12-06T16:00:53,508 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/A in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:53,508 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/bb63e022a32843fe805a2fbf8883c106, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/87cdfc890b9c49e495ed4c3fe7dfbefb, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/c271b18083e242de858228aeb018595b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/5b7c2b55931946a48a219bfa0a3aea80] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=49.1 K 2024-12-06T16:00:53,509 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb63e022a32843fe805a2fbf8883c106, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733500849162 2024-12-06T16:00:53,509 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting da1f1c647c004424ae51e876737812c5, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733500849162 2024-12-06T16:00:53,511 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87cdfc890b9c49e495ed4c3fe7dfbefb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1733500849883 2024-12-06T16:00:53,511 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 74dc70a38f474570bffac688ae22d836, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1733500849883 2024-12-06T16:00:53,513 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting c271b18083e242de858228aeb018595b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1733500850570 2024-12-06T16:00:53,513 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting e1523ea29bbd4072ac809a383193e80d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1733500850570 2024-12-06T16:00:53,513 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b7c2b55931946a48a219bfa0a3aea80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=517, earliestPutTs=1733500851770 2024-12-06T16:00:53,513 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 34d521cd0dc2468db37d03f31224da2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=517, earliestPutTs=1733500851770 2024-12-06T16:00:53,542 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#A#compaction#108 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:53,543 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#B#compaction#109 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:53,543 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/8b0f789c5fe24448b63bddacd63a7472 is 50, key is test_row_0/A:col10/1733500852946/Put/seqid=0 2024-12-06T16:00:53,543 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/00e0761ab0d547bab7446482e3b180e0 is 50, key is test_row_0/B:col10/1733500852946/Put/seqid=0 2024-12-06T16:00:53,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741945_1121 (size=13527) 2024-12-06T16:00:53,560 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/8b0f789c5fe24448b63bddacd63a7472 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/8b0f789c5fe24448b63bddacd63a7472 2024-12-06T16:00:53,570 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/A of 81bfe42898e92490dcd2dea3186de72d into 8b0f789c5fe24448b63bddacd63a7472(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:53,570 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:53,571 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/A, priority=12, startTime=1733500853504; duration=0sec 2024-12-06T16:00:53,571 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:00:53,571 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:A 2024-12-06T16:00:53,571 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T16:00:53,573 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50294 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T16:00:53,573 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/C is initiating minor compaction (all files) 2024-12-06T16:00:53,573 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/C in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:53,578 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/a40c7e21c99d4b02aa7ae4e667d38cc7, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/c7d2168c0493421eb85012fdd10c1af3, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/6cb91ab09bc84f228e839b9dba15f645, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/7b9600ffe7e24320bd4ecfb313efed87] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=49.1 K 2024-12-06T16:00:53,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741946_1122 (size=13527) 2024-12-06T16:00:53,579 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting a40c7e21c99d4b02aa7ae4e667d38cc7, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733500849162 2024-12-06T16:00:53,579 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7d2168c0493421eb85012fdd10c1af3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1733500849883 2024-12-06T16:00:53,580 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cb91ab09bc84f228e839b9dba15f645, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1733500850570 2024-12-06T16:00:53,581 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b9600ffe7e24320bd4ecfb313efed87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=517, earliestPutTs=1733500851770 2024-12-06T16:00:53,589 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/00e0761ab0d547bab7446482e3b180e0 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/00e0761ab0d547bab7446482e3b180e0 2024-12-06T16:00:53,598 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/B of 81bfe42898e92490dcd2dea3186de72d into 00e0761ab0d547bab7446482e3b180e0(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:53,599 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:53,599 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/B, priority=12, startTime=1733500853504; duration=0sec 2024-12-06T16:00:53,599 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:53,599 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:B 2024-12-06T16:00:53,613 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#C#compaction#110 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:53,614 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/173effb815db4c5db9a44d840b55fb5b is 50, key is test_row_0/C:col10/1733500852946/Put/seqid=0 2024-12-06T16:00:53,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741947_1123 (size=13527) 2024-12-06T16:00:53,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:53,649 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:00:53,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81bfe42898e92490dcd2dea3186de72d 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-06T16:00:53,657 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/173effb815db4c5db9a44d840b55fb5b as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/173effb815db4c5db9a44d840b55fb5b 2024-12-06T16:00:53,666 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/C of 81bfe42898e92490dcd2dea3186de72d into 173effb815db4c5db9a44d840b55fb5b(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:53,666 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:53,666 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/C, priority=12, startTime=1733500853505; duration=0sec 2024-12-06T16:00:53,666 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:53,666 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:C 2024-12-06T16:00:53,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/e16bd5cb0a59409d9fba7f7d1fd88a17 is 50, key is test_row_0/A:col10/1733500853646/Put/seqid=0 2024-12-06T16:00:53,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:53,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500913682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:53,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:53,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:53,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:53,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:53,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500913688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:53,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:53,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:53,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:53,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:53,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500913688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:53,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:53,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:53,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:53,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741948_1124 (size=12301) 2024-12-06T16:00:53,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:53,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500913792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:53,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:53,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500913803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:53,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:53,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:53,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:53,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:53,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500913812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:53,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:53,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:53,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:54,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:54,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500914002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:54,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:54,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:54,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:54,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:54,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500914012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:54,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:54,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500914021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:54,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:54,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:54,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:54,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/e16bd5cb0a59409d9fba7f7d1fd88a17 2024-12-06T16:00:54,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/24e80e8e80364e03bc462cfbf9970f36 is 50, key is test_row_0/B:col10/1733500853646/Put/seqid=0 2024-12-06T16:00:54,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741949_1125 (size=12301) 2024-12-06T16:00:54,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40738 deadline: 1733500914191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:54,196 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 85bef17d9292,46105,1733500827732 for 'test_row_2' in 81bfe42898e92490dcd2dea3186de72d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T16:00:54,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:54,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:54,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:54,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40752 deadline: 1733500914202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:54,207 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 85bef17d9292,46105,1733500827732 for 'test_row_2' in 81bfe42898e92490dcd2dea3186de72d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T16:00:54,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:54,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:54,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:54,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40704 deadline: 1733500914312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:54,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:54,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:54,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:54,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40726 deadline: 1733500914322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:54,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:54,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:54,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:54,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:00:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1733500914332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 2024-12-06T16:00:54,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:54,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:00:54,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:00:54,435 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-06T16:00:54,435 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-06T16:00:54,435 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:00:54,435 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:00:54,436 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-06T16:00:54,436 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:00:54,440 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:00:54,440 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:00:54,440 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:00:54,441 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:00:54,441 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:00:54,441 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:00:54,441 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:00:54,441 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:00:54,441 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:00:54,443 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-06T16:00:54,443 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:00:54,443 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:00:54,443 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:00:54,443 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:00:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-06T16:00:54,532 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:00:54,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/24e80e8e80364e03bc462cfbf9970f36 2024-12-06T16:00:54,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/dc57db6371ed4098b90e625265e3e1f8 is 50, key is test_row_0/C:col10/1733500853646/Put/seqid=0 2024-12-06T16:00:54,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741950_1126 (size=12301) 2024-12-06T16:00:54,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/dc57db6371ed4098b90e625265e3e1f8 2024-12-06T16:00:54,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/e16bd5cb0a59409d9fba7f7d1fd88a17 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/e16bd5cb0a59409d9fba7f7d1fd88a17 2024-12-06T16:00:54,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/e16bd5cb0a59409d9fba7f7d1fd88a17, entries=150, sequenceid=541, filesize=12.0 K 2024-12-06T16:00:54,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/24e80e8e80364e03bc462cfbf9970f36 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/24e80e8e80364e03bc462cfbf9970f36 2024-12-06T16:00:54,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/24e80e8e80364e03bc462cfbf9970f36, entries=150, sequenceid=541, filesize=12.0 K 2024-12-06T16:00:54,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/dc57db6371ed4098b90e625265e3e1f8 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/dc57db6371ed4098b90e625265e3e1f8 2024-12-06T16:00:54,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/dc57db6371ed4098b90e625265e3e1f8, entries=150, sequenceid=541, filesize=12.0 K 2024-12-06T16:00:54,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 81bfe42898e92490dcd2dea3186de72d in 959ms, sequenceid=541, compaction requested=false 2024-12-06T16:00:54,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:00:54,824 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:00:54,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81bfe42898e92490dcd2dea3186de72d 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-06T16:00:54,824 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-06T16:00:54,824 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:00:54,824 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:00:54,825 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:00:54,825 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:00:54,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/9755f14b9ea74f5faeb32aec63f55d92 is 50, key is test_row_0/A:col10/1733500854822/Put/seqid=0 2024-12-06T16:00:54,833 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-06T16:00:54,833 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:00:54,833 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:00:54,833 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:00:54,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741951_1127 (size=12301) 2024-12-06T16:00:54,834 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:00:54,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/9755f14b9ea74f5faeb32aec63f55d92 2024-12-06T16:00:54,844 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-06T16:00:54,844 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:00:54,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/6725c6cab3e04a5da4224f96f070edb5 is 50, key is test_row_0/B:col10/1733500854822/Put/seqid=0 2024-12-06T16:00:54,844 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:00:54,844 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:00:54,844 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:00:54,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741952_1128 (size=12301) 2024-12-06T16:00:55,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/6725c6cab3e04a5da4224f96f070edb5 2024-12-06T16:00:55,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/43a2a164ebf94fbcad793a1d63c1d8d5 is 50, key is test_row_0/C:col10/1733500854822/Put/seqid=0 2024-12-06T16:00:55,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741953_1129 (size=12301) 2024-12-06T16:00:55,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/43a2a164ebf94fbcad793a1d63c1d8d5 2024-12-06T16:00:55,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/9755f14b9ea74f5faeb32aec63f55d92 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/9755f14b9ea74f5faeb32aec63f55d92 2024-12-06T16:00:55,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/9755f14b9ea74f5faeb32aec63f55d92, entries=150, sequenceid=558, filesize=12.0 K 2024-12-06T16:00:55,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/6725c6cab3e04a5da4224f96f070edb5 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/6725c6cab3e04a5da4224f96f070edb5 2024-12-06T16:00:55,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/6725c6cab3e04a5da4224f96f070edb5, entries=150, sequenceid=558, filesize=12.0 K 2024-12-06T16:00:55,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/43a2a164ebf94fbcad793a1d63c1d8d5 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/43a2a164ebf94fbcad793a1d63c1d8d5 2024-12-06T16:00:55,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/43a2a164ebf94fbcad793a1d63c1d8d5, entries=150, sequenceid=558, filesize=12.0 K 2024-12-06T16:00:55,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=13.42 KB/13740 for 81bfe42898e92490dcd2dea3186de72d in 868ms, sequenceid=558, compaction requested=true 2024-12-06T16:00:55,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:55,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:00:55,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:55,692 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:00:55,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:00:55,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:55,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81bfe42898e92490dcd2dea3186de72d:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:00:55,693 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:00:55,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:00:55,694 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38129 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:00:55,694 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38129 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:00:55,694 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/B is initiating minor compaction (all files) 2024-12-06T16:00:55,694 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/A is initiating minor compaction (all files) 2024-12-06T16:00:55,694 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/A in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:55,694 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/B in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:55,694 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/00e0761ab0d547bab7446482e3b180e0, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/24e80e8e80364e03bc462cfbf9970f36, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/6725c6cab3e04a5da4224f96f070edb5] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=37.2 K 2024-12-06T16:00:55,694 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/8b0f789c5fe24448b63bddacd63a7472, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/e16bd5cb0a59409d9fba7f7d1fd88a17, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/9755f14b9ea74f5faeb32aec63f55d92] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=37.2 K 2024-12-06T16:00:55,694 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b0f789c5fe24448b63bddacd63a7472, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=517, earliestPutTs=1733500851770 2024-12-06T16:00:55,694 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 00e0761ab0d547bab7446482e3b180e0, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=517, earliestPutTs=1733500851770 2024-12-06T16:00:55,695 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting e16bd5cb0a59409d9fba7f7d1fd88a17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1733500852987 2024-12-06T16:00:55,695 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 24e80e8e80364e03bc462cfbf9970f36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1733500852987 2024-12-06T16:00:55,695 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9755f14b9ea74f5faeb32aec63f55d92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=558, earliestPutTs=1733500853675 2024-12-06T16:00:55,695 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 6725c6cab3e04a5da4224f96f070edb5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=558, earliestPutTs=1733500853675 2024-12-06T16:00:55,705 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#B#compaction#117 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:55,705 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#A#compaction#118 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:55,705 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/9f8bd252ebc04dda836c4e6d8d927f19 is 50, key is test_row_0/B:col10/1733500854822/Put/seqid=0 2024-12-06T16:00:55,706 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/90a3286cdbb44ce6a6dc9abfcf414bae is 50, key is test_row_0/A:col10/1733500854822/Put/seqid=0 2024-12-06T16:00:55,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741955_1131 (size=13629) 2024-12-06T16:00:55,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741954_1130 (size=13629) 2024-12-06T16:00:55,827 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-06T16:00:56,116 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/9f8bd252ebc04dda836c4e6d8d927f19 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/9f8bd252ebc04dda836c4e6d8d927f19 2024-12-06T16:00:56,116 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/90a3286cdbb44ce6a6dc9abfcf414bae as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/90a3286cdbb44ce6a6dc9abfcf414bae 2024-12-06T16:00:56,122 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/B of 81bfe42898e92490dcd2dea3186de72d into 9f8bd252ebc04dda836c4e6d8d927f19(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:56,122 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/A of 81bfe42898e92490dcd2dea3186de72d into 90a3286cdbb44ce6a6dc9abfcf414bae(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:56,123 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:56,123 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:56,123 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/B, priority=13, startTime=1733500855692; duration=0sec 2024-12-06T16:00:56,123 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/A, priority=13, startTime=1733500855692; duration=0sec 2024-12-06T16:00:56,123 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:00:56,123 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:00:56,123 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:A 2024-12-06T16:00:56,123 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:B 2024-12-06T16:00:56,123 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:00:56,124 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38129 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:00:56,124 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 81bfe42898e92490dcd2dea3186de72d/C is initiating minor compaction (all files) 2024-12-06T16:00:56,124 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81bfe42898e92490dcd2dea3186de72d/C in TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:00:56,124 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/173effb815db4c5db9a44d840b55fb5b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/dc57db6371ed4098b90e625265e3e1f8, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/43a2a164ebf94fbcad793a1d63c1d8d5] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp, totalSize=37.2 K 2024-12-06T16:00:56,125 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 173effb815db4c5db9a44d840b55fb5b, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=517, earliestPutTs=1733500851770 2024-12-06T16:00:56,125 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc57db6371ed4098b90e625265e3e1f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1733500852987 2024-12-06T16:00:56,125 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43a2a164ebf94fbcad793a1d63c1d8d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=558, earliestPutTs=1733500853675 2024-12-06T16:00:56,133 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81bfe42898e92490dcd2dea3186de72d#C#compaction#119 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:00:56,134 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/626f6cf14324443c9daa4b0ce894a00c is 50, key is test_row_0/C:col10/1733500854822/Put/seqid=0 2024-12-06T16:00:56,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741956_1132 (size=13629) 2024-12-06T16:00:56,545 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/626f6cf14324443c9daa4b0ce894a00c as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/626f6cf14324443c9daa4b0ce894a00c 2024-12-06T16:00:56,550 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81bfe42898e92490dcd2dea3186de72d/C of 81bfe42898e92490dcd2dea3186de72d into 626f6cf14324443c9daa4b0ce894a00c(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:00:56,551 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81bfe42898e92490dcd2dea3186de72d: 2024-12-06T16:00:56,551 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., storeName=81bfe42898e92490dcd2dea3186de72d/C, priority=13, startTime=1733500855693; duration=0sec 2024-12-06T16:00:56,551 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:00:56,551 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81bfe42898e92490dcd2dea3186de72d:C 2024-12-06T16:01:04,274 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-06T16:01:04,274 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:01:04,274 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:04,274 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:04,274 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:01:04,324 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-06T16:01:04,324 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:01:04,324 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:04,325 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:04,325 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:01:04,325 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-06T16:01:04,325 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-12-06T16:01:04,325 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-12-06T16:01:04,325 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-12-06T16:01:04,325 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-12-06T16:01:04,325 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 113 2024-12-06T16:01:04,325 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-06T16:01:04,325 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2668 2024-12-06T16:01:04,325 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2651 2024-12-06T16:01:04,325 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-06T16:01:04,325 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1230 2024-12-06T16:01:04,326 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3673 rows 2024-12-06T16:01:04,326 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1234 2024-12-06T16:01:04,326 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3692 rows 2024-12-06T16:01:04,326 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T16:01:04,326 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T16:01:04,326 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:04,326 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:04,327 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:01:04,331 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-06T16:01:04,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-06T16:01:04,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-06T16:01:04,340 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733500864339"}]},"ts":"1733500864339"} 2024-12-06T16:01:04,342 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-06T16:01:04,342 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-06T16:01:04,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-06T16:01:04,351 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81bfe42898e92490dcd2dea3186de72d, UNASSIGN}] 2024-12-06T16:01:04,352 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81bfe42898e92490dcd2dea3186de72d, UNASSIGN 2024-12-06T16:01:04,354 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=81bfe42898e92490dcd2dea3186de72d, regionState=CLOSING, regionLocation=85bef17d9292,46105,1733500827732 2024-12-06T16:01:04,356 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=81bfe42898e92490dcd2dea3186de72d, UNASSIGN because future has completed 2024-12-06T16:01:04,356 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-06T16:01:04,357 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; CloseRegionProcedure 81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732}] 2024-12-06T16:01:04,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-06T16:01:04,515 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(122): Close 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:01:04,515 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-06T16:01:04,516 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1722): Closing 81bfe42898e92490dcd2dea3186de72d, disabling compactions & flushes 2024-12-06T16:01:04,516 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:01:04,516 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:01:04,516 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. after waiting 0 ms 2024-12-06T16:01:04,516 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:01:04,516 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(2902): Flushing 81bfe42898e92490dcd2dea3186de72d 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-12-06T16:01:04,521 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/4f30d5a4f2744f4482971d4590fd6ff1 is 50, key is test_row_0/A:col10/1733500854832/Put/seqid=0 2024-12-06T16:01:04,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741957_1133 (size=9857) 2024-12-06T16:01:04,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-06T16:01:04,928 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=568 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/4f30d5a4f2744f4482971d4590fd6ff1 2024-12-06T16:01:04,937 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/e6dbb054ec7546ef820e8e76ff28b63c is 50, key is test_row_0/B:col10/1733500854832/Put/seqid=0 2024-12-06T16:01:04,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741958_1134 (size=9857) 2024-12-06T16:01:04,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-06T16:01:05,344 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=568 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/e6dbb054ec7546ef820e8e76ff28b63c 2024-12-06T16:01:05,352 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/d700897ec2374d449a78c71751e5d301 is 50, key is test_row_0/C:col10/1733500854832/Put/seqid=0 2024-12-06T16:01:05,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741959_1135 (size=9857) 2024-12-06T16:01:05,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-06T16:01:05,757 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=568 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/d700897ec2374d449a78c71751e5d301 2024-12-06T16:01:05,763 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/A/4f30d5a4f2744f4482971d4590fd6ff1 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/4f30d5a4f2744f4482971d4590fd6ff1 2024-12-06T16:01:05,767 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/4f30d5a4f2744f4482971d4590fd6ff1, entries=100, sequenceid=568, filesize=9.6 K 2024-12-06T16:01:05,768 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/B/e6dbb054ec7546ef820e8e76ff28b63c as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/e6dbb054ec7546ef820e8e76ff28b63c 2024-12-06T16:01:05,773 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/e6dbb054ec7546ef820e8e76ff28b63c, entries=100, sequenceid=568, filesize=9.6 K 2024-12-06T16:01:05,774 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/.tmp/C/d700897ec2374d449a78c71751e5d301 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/d700897ec2374d449a78c71751e5d301 2024-12-06T16:01:05,778 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/d700897ec2374d449a78c71751e5d301, entries=100, sequenceid=568, filesize=9.6 K 2024-12-06T16:01:05,779 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 81bfe42898e92490dcd2dea3186de72d in 1263ms, sequenceid=568, compaction requested=false 2024-12-06T16:01:05,780 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/9c86b711d19445ea90d0dac966b7547d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/2ad7b5dd0a7e4bdca8ff4cfe3a99ce73, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/c1437c485fa648a6a170ea8ad91d4d0b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/cde848b722cf4da98fd5959d859b98ab, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/30ea301006064d6fb56491015907ad61, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/cb991b5891194564b94ba2d125c01529, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/41b21c1c0f1b4f72ae2efc2cc1c613bb, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/e1452d8fd06c460e877240a0272ad975, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/f262ded6286847fdbf7fb88bdaa5e4f7, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/b484b8d1e1cb40cf81169bd0821dff04, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/3896fd23dd4b4c4e96773db34dfb649f, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/265e8fa92c4d4dcda47570cff1f05208, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/6f9fb8e2999943078f59e748f5bd95c6, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/a2a9179d493b4963b3ad249951f28f4e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/68e4f52a7f574aae8aa6e82ecb20b6d8, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/0a24ed42627d42b4af34e6d30f666600, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/721cbb5637a44ae8a7bde7a4a0f0569b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/1392ee1eb54541e0a91cc0899bfe8c60, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/66b953419652443984127f8ea327a8ad, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/9bd7823797a54adf91806de1a088324c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/2cc3b666af0d42c697ce818e792c4127, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/fea31613cc3b4efb85affccc0fe7baf6, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/d10de112330e45b4afb88afdd34db7d1, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/0e241960a7524874999038c6461e7f97, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/7e5bf8e1a19644288d8b94a3f7b113e5, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/9d7fe631002f4d0c8b1059c5c5046322, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/9ce1abc1b8b643668155df102f8ba56c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/42ed87278f284c75a13b7a971388f366, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/692c0170d2244846a58a055ac511b6d7, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/5c0caf8dd8334fb5bcf03bb3a3978cd5, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/818e4bedca64434cb97f63e35dbfa70e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/1f7986ab167f4b1fad9952be1f689e3e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/bb63e022a32843fe805a2fbf8883c106, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/87cdfc890b9c49e495ed4c3fe7dfbefb, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/c271b18083e242de858228aeb018595b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/8b0f789c5fe24448b63bddacd63a7472, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/5b7c2b55931946a48a219bfa0a3aea80, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/e16bd5cb0a59409d9fba7f7d1fd88a17, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/9755f14b9ea74f5faeb32aec63f55d92] to archive 2024-12-06T16:01:05,783 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T16:01:05,789 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/9c86b711d19445ea90d0dac966b7547d to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/9c86b711d19445ea90d0dac966b7547d 2024-12-06T16:01:05,791 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/2ad7b5dd0a7e4bdca8ff4cfe3a99ce73 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/2ad7b5dd0a7e4bdca8ff4cfe3a99ce73 2024-12-06T16:01:05,793 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/c1437c485fa648a6a170ea8ad91d4d0b to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/c1437c485fa648a6a170ea8ad91d4d0b 2024-12-06T16:01:05,794 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/cde848b722cf4da98fd5959d859b98ab to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/cde848b722cf4da98fd5959d859b98ab 2024-12-06T16:01:05,796 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/30ea301006064d6fb56491015907ad61 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/30ea301006064d6fb56491015907ad61 2024-12-06T16:01:05,798 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/cb991b5891194564b94ba2d125c01529 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/cb991b5891194564b94ba2d125c01529 2024-12-06T16:01:05,799 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/41b21c1c0f1b4f72ae2efc2cc1c613bb to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/41b21c1c0f1b4f72ae2efc2cc1c613bb 2024-12-06T16:01:05,800 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/e1452d8fd06c460e877240a0272ad975 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/e1452d8fd06c460e877240a0272ad975 2024-12-06T16:01:05,802 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/f262ded6286847fdbf7fb88bdaa5e4f7 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/f262ded6286847fdbf7fb88bdaa5e4f7 2024-12-06T16:01:05,803 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/b484b8d1e1cb40cf81169bd0821dff04 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/b484b8d1e1cb40cf81169bd0821dff04 2024-12-06T16:01:05,805 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/3896fd23dd4b4c4e96773db34dfb649f to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/3896fd23dd4b4c4e96773db34dfb649f 2024-12-06T16:01:05,806 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/265e8fa92c4d4dcda47570cff1f05208 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/265e8fa92c4d4dcda47570cff1f05208 2024-12-06T16:01:05,807 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/6f9fb8e2999943078f59e748f5bd95c6 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/6f9fb8e2999943078f59e748f5bd95c6 2024-12-06T16:01:05,809 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/a2a9179d493b4963b3ad249951f28f4e to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/a2a9179d493b4963b3ad249951f28f4e 2024-12-06T16:01:05,810 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/68e4f52a7f574aae8aa6e82ecb20b6d8 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/68e4f52a7f574aae8aa6e82ecb20b6d8 2024-12-06T16:01:05,811 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/0a24ed42627d42b4af34e6d30f666600 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/0a24ed42627d42b4af34e6d30f666600 2024-12-06T16:01:05,813 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/721cbb5637a44ae8a7bde7a4a0f0569b to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/721cbb5637a44ae8a7bde7a4a0f0569b 2024-12-06T16:01:05,814 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/1392ee1eb54541e0a91cc0899bfe8c60 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/1392ee1eb54541e0a91cc0899bfe8c60 2024-12-06T16:01:05,815 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/66b953419652443984127f8ea327a8ad to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/66b953419652443984127f8ea327a8ad 2024-12-06T16:01:05,817 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/9bd7823797a54adf91806de1a088324c to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/9bd7823797a54adf91806de1a088324c 2024-12-06T16:01:05,818 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/2cc3b666af0d42c697ce818e792c4127 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/2cc3b666af0d42c697ce818e792c4127 2024-12-06T16:01:05,820 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/fea31613cc3b4efb85affccc0fe7baf6 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/fea31613cc3b4efb85affccc0fe7baf6 2024-12-06T16:01:05,821 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/d10de112330e45b4afb88afdd34db7d1 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/d10de112330e45b4afb88afdd34db7d1 2024-12-06T16:01:05,822 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/0e241960a7524874999038c6461e7f97 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/0e241960a7524874999038c6461e7f97 2024-12-06T16:01:05,824 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/7e5bf8e1a19644288d8b94a3f7b113e5 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/7e5bf8e1a19644288d8b94a3f7b113e5 2024-12-06T16:01:05,826 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/9d7fe631002f4d0c8b1059c5c5046322 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/9d7fe631002f4d0c8b1059c5c5046322 2024-12-06T16:01:05,827 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/9ce1abc1b8b643668155df102f8ba56c to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/9ce1abc1b8b643668155df102f8ba56c 2024-12-06T16:01:05,829 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/42ed87278f284c75a13b7a971388f366 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/42ed87278f284c75a13b7a971388f366 2024-12-06T16:01:05,831 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/692c0170d2244846a58a055ac511b6d7 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/692c0170d2244846a58a055ac511b6d7 2024-12-06T16:01:05,832 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/5c0caf8dd8334fb5bcf03bb3a3978cd5 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/5c0caf8dd8334fb5bcf03bb3a3978cd5 2024-12-06T16:01:05,834 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/818e4bedca64434cb97f63e35dbfa70e to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/818e4bedca64434cb97f63e35dbfa70e 2024-12-06T16:01:05,835 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/1f7986ab167f4b1fad9952be1f689e3e to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/1f7986ab167f4b1fad9952be1f689e3e 2024-12-06T16:01:05,837 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/bb63e022a32843fe805a2fbf8883c106 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/bb63e022a32843fe805a2fbf8883c106 2024-12-06T16:01:05,839 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/87cdfc890b9c49e495ed4c3fe7dfbefb to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/87cdfc890b9c49e495ed4c3fe7dfbefb 2024-12-06T16:01:05,840 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/c271b18083e242de858228aeb018595b to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/c271b18083e242de858228aeb018595b 2024-12-06T16:01:05,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/8b0f789c5fe24448b63bddacd63a7472 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/8b0f789c5fe24448b63bddacd63a7472 2024-12-06T16:01:05,843 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/5b7c2b55931946a48a219bfa0a3aea80 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/5b7c2b55931946a48a219bfa0a3aea80 2024-12-06T16:01:05,845 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/e16bd5cb0a59409d9fba7f7d1fd88a17 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/e16bd5cb0a59409d9fba7f7d1fd88a17 2024-12-06T16:01:05,846 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/9755f14b9ea74f5faeb32aec63f55d92 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/9755f14b9ea74f5faeb32aec63f55d92 2024-12-06T16:01:05,866 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/d3b71cfa7747430e93c2dfce60fa05ee, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/20b7171476974be3afffa1704fa2efe8, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/06dbeaedd7054946a34244e746431f4f, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/abd3c53799c946e993968783d50bf804, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/24e3bcf1530846458a163e8167fdcdea, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/039ff872c56a45f5b20096f94c86b698, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/5e7b4c5e88774e3c93d47372c963113b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/e1a39ae9b9df408da3e49e0aa600badf, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/7cb03216a4e244b4826692ff0485fc20, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/98d0cd4405c94303b4ec0668a3f4f0c2, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/99e24d2fdf5142d5a419529da13439bc, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/8e66120fe00e4368a81a4a665c9d3271, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/474f5f44b33a48cf9d76b4ee7612d6bb, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/83534f3cd9c04025937fff0b4706d81f, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/017203f6d03c4d3ca4325eed201e0105, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/e0c3fb20c1654f3d99cb527ad014f964, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/2ced15e61f184e47a3c714f5dcf6c11e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/371c112a97ac482c950dc4b39fe771c5, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/1bf26d526b95454890a854462486d30a, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/927d02c30967487bb8d95c7a75cb0ce7, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/c1207aff83784fc1b6f183b098590789, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/d0c57682738c4f0986bad92806c85ac5, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/19fe4af070cf43c6a66f586e0c8c1480, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/28dcd6faeb3f43648fe5ea4cac4f9ed2, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/293417992d714250be624998ebbc1b4c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/1f201f9f69c645e2be1ec2a0dbe2b645, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/6a7017dd29d148d994eebf69b88e4538, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/2ced8238925844049b539f8441fd4bc8, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/ff25f05bc08b4512b2c6d92c6fbe8836, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/1fa2c75a8fea4157b93ab23a13655489, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/5c1b93dd729d4588bf8c5399d6ac776e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/da1f1c647c004424ae51e876737812c5, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/c9dd52db4bef4139ac7e481feee7150b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/74dc70a38f474570bffac688ae22d836, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/e1523ea29bbd4072ac809a383193e80d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/00e0761ab0d547bab7446482e3b180e0, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/34d521cd0dc2468db37d03f31224da2e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/24e80e8e80364e03bc462cfbf9970f36, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/6725c6cab3e04a5da4224f96f070edb5] to archive 2024-12-06T16:01:05,868 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T16:01:05,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/d3b71cfa7747430e93c2dfce60fa05ee to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/d3b71cfa7747430e93c2dfce60fa05ee 2024-12-06T16:01:05,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/20b7171476974be3afffa1704fa2efe8 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/20b7171476974be3afffa1704fa2efe8 2024-12-06T16:01:05,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/06dbeaedd7054946a34244e746431f4f to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/06dbeaedd7054946a34244e746431f4f 2024-12-06T16:01:05,879 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/abd3c53799c946e993968783d50bf804 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/abd3c53799c946e993968783d50bf804 2024-12-06T16:01:05,880 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/24e3bcf1530846458a163e8167fdcdea to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/24e3bcf1530846458a163e8167fdcdea 2024-12-06T16:01:05,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/039ff872c56a45f5b20096f94c86b698 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/039ff872c56a45f5b20096f94c86b698 2024-12-06T16:01:05,883 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/5e7b4c5e88774e3c93d47372c963113b to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/5e7b4c5e88774e3c93d47372c963113b 2024-12-06T16:01:05,885 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/e1a39ae9b9df408da3e49e0aa600badf to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/e1a39ae9b9df408da3e49e0aa600badf 2024-12-06T16:01:05,886 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/7cb03216a4e244b4826692ff0485fc20 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/7cb03216a4e244b4826692ff0485fc20 2024-12-06T16:01:05,887 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/98d0cd4405c94303b4ec0668a3f4f0c2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/98d0cd4405c94303b4ec0668a3f4f0c2 2024-12-06T16:01:05,889 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/99e24d2fdf5142d5a419529da13439bc to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/99e24d2fdf5142d5a419529da13439bc 2024-12-06T16:01:05,890 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/8e66120fe00e4368a81a4a665c9d3271 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/8e66120fe00e4368a81a4a665c9d3271 2024-12-06T16:01:05,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/474f5f44b33a48cf9d76b4ee7612d6bb to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/474f5f44b33a48cf9d76b4ee7612d6bb 2024-12-06T16:01:05,900 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/83534f3cd9c04025937fff0b4706d81f to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/83534f3cd9c04025937fff0b4706d81f 2024-12-06T16:01:05,901 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/017203f6d03c4d3ca4325eed201e0105 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/017203f6d03c4d3ca4325eed201e0105 2024-12-06T16:01:05,902 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/e0c3fb20c1654f3d99cb527ad014f964 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/e0c3fb20c1654f3d99cb527ad014f964 2024-12-06T16:01:05,903 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/2ced15e61f184e47a3c714f5dcf6c11e to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/2ced15e61f184e47a3c714f5dcf6c11e 2024-12-06T16:01:05,913 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/371c112a97ac482c950dc4b39fe771c5 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/371c112a97ac482c950dc4b39fe771c5 2024-12-06T16:01:05,914 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/1bf26d526b95454890a854462486d30a to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/1bf26d526b95454890a854462486d30a 2024-12-06T16:01:05,916 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/927d02c30967487bb8d95c7a75cb0ce7 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/927d02c30967487bb8d95c7a75cb0ce7 2024-12-06T16:01:05,917 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/c1207aff83784fc1b6f183b098590789 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/c1207aff83784fc1b6f183b098590789 2024-12-06T16:01:05,919 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/d0c57682738c4f0986bad92806c85ac5 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/d0c57682738c4f0986bad92806c85ac5 2024-12-06T16:01:05,921 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/19fe4af070cf43c6a66f586e0c8c1480 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/19fe4af070cf43c6a66f586e0c8c1480 2024-12-06T16:01:05,922 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/28dcd6faeb3f43648fe5ea4cac4f9ed2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/28dcd6faeb3f43648fe5ea4cac4f9ed2 2024-12-06T16:01:05,924 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/293417992d714250be624998ebbc1b4c to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/293417992d714250be624998ebbc1b4c 2024-12-06T16:01:05,925 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/1f201f9f69c645e2be1ec2a0dbe2b645 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/1f201f9f69c645e2be1ec2a0dbe2b645 2024-12-06T16:01:05,927 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/6a7017dd29d148d994eebf69b88e4538 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/6a7017dd29d148d994eebf69b88e4538 2024-12-06T16:01:05,928 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/2ced8238925844049b539f8441fd4bc8 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/2ced8238925844049b539f8441fd4bc8 2024-12-06T16:01:05,930 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/ff25f05bc08b4512b2c6d92c6fbe8836 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/ff25f05bc08b4512b2c6d92c6fbe8836 2024-12-06T16:01:05,931 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/1fa2c75a8fea4157b93ab23a13655489 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/1fa2c75a8fea4157b93ab23a13655489 2024-12-06T16:01:05,933 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/5c1b93dd729d4588bf8c5399d6ac776e to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/5c1b93dd729d4588bf8c5399d6ac776e 2024-12-06T16:01:05,940 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/da1f1c647c004424ae51e876737812c5 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/da1f1c647c004424ae51e876737812c5 2024-12-06T16:01:05,941 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/c9dd52db4bef4139ac7e481feee7150b to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/c9dd52db4bef4139ac7e481feee7150b 2024-12-06T16:01:05,942 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/74dc70a38f474570bffac688ae22d836 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/74dc70a38f474570bffac688ae22d836 2024-12-06T16:01:05,943 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/e1523ea29bbd4072ac809a383193e80d to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/e1523ea29bbd4072ac809a383193e80d 2024-12-06T16:01:05,944 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/00e0761ab0d547bab7446482e3b180e0 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/00e0761ab0d547bab7446482e3b180e0 2024-12-06T16:01:05,946 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/34d521cd0dc2468db37d03f31224da2e to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/34d521cd0dc2468db37d03f31224da2e 2024-12-06T16:01:05,947 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/24e80e8e80364e03bc462cfbf9970f36 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/24e80e8e80364e03bc462cfbf9970f36 2024-12-06T16:01:05,948 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/6725c6cab3e04a5da4224f96f070edb5 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/6725c6cab3e04a5da4224f96f070edb5 2024-12-06T16:01:05,950 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/780815a6351a4b848606dcdc5d037987, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/2b86102c03074554825a233e07750076, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/18c416650918464da4bba863a49fcee3, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/423a6cc5feee435cba032de14b69c48c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/f37ed2b1c7f142deab2b201258c79592, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/7fa9981806254bbe8ab1f7eaf3cb064d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/698244ac2ca945e5be2ec6590800a8e3, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/ce33412f769c488ea6d790e537c2db76, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/cbfd028b010b4a918b7d92182b2a3206, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/79a5c2e7c0594e9d82535d8aff691a60, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/84125e8896524524a9ee49d6ef749d10, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/c95173f4df9b4784a49552aa6fd47bf3, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/52eea21c14fd4bc9ad27b6c944029622, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/0f827ce12fa64c96beadfb3595973869, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/c0c937516075446085bd9d21c2cc7d0b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/8956e325d48f4f2691df11e757efc359, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/e8a0a42aeb224ecb826258eb878db948, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/40cfd0cc6c8f47809c2953b7b1e4ddc4, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/08c8742946864c9f9d6ac5f9ea3ed45f, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/09f6ad844e984d62a2e413307fc65cf8, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/56ee5c2f5fc04f40a0b753b07fffaceb, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/14616e15b1a34ffbb816792eab8ddcc0, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/842b803c611e4ec5b4a0dbaf8679e63b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/92104cae7974403b8b492cd0007c0c73, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/da8d4fafaaa84b6a9fc64bfaf41779f3, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/bd2c3b8609fc4af689ac6866265fa3d4, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/1c2c85f9127b4ecb95ceec2e599b6f41, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/42fe32e9d0024881bafcabf4d91f8e0b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/1d63aadd7d4743048773b6820ac28d5a, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/022e94ec72014382a1d62f48bba40b03, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/f4b0f41751164b72add566aae08bb760, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/a40c7e21c99d4b02aa7ae4e667d38cc7, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/e56c148a815f42cdaa8d34e7beed587a, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/c7d2168c0493421eb85012fdd10c1af3, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/6cb91ab09bc84f228e839b9dba15f645, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/173effb815db4c5db9a44d840b55fb5b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/7b9600ffe7e24320bd4ecfb313efed87, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/dc57db6371ed4098b90e625265e3e1f8, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/43a2a164ebf94fbcad793a1d63c1d8d5] to archive 2024-12-06T16:01:05,952 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T16:01:05,953 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/780815a6351a4b848606dcdc5d037987 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/780815a6351a4b848606dcdc5d037987 2024-12-06T16:01:05,955 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/2b86102c03074554825a233e07750076 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/2b86102c03074554825a233e07750076 2024-12-06T16:01:05,956 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/18c416650918464da4bba863a49fcee3 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/18c416650918464da4bba863a49fcee3 2024-12-06T16:01:05,957 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/423a6cc5feee435cba032de14b69c48c to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/423a6cc5feee435cba032de14b69c48c 2024-12-06T16:01:05,959 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/f37ed2b1c7f142deab2b201258c79592 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/f37ed2b1c7f142deab2b201258c79592 2024-12-06T16:01:05,960 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/7fa9981806254bbe8ab1f7eaf3cb064d to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/7fa9981806254bbe8ab1f7eaf3cb064d 2024-12-06T16:01:05,965 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/698244ac2ca945e5be2ec6590800a8e3 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/698244ac2ca945e5be2ec6590800a8e3 2024-12-06T16:01:05,966 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/ce33412f769c488ea6d790e537c2db76 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/ce33412f769c488ea6d790e537c2db76 2024-12-06T16:01:05,968 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/cbfd028b010b4a918b7d92182b2a3206 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/cbfd028b010b4a918b7d92182b2a3206 2024-12-06T16:01:05,971 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/79a5c2e7c0594e9d82535d8aff691a60 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/79a5c2e7c0594e9d82535d8aff691a60 2024-12-06T16:01:05,972 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/84125e8896524524a9ee49d6ef749d10 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/84125e8896524524a9ee49d6ef749d10 2024-12-06T16:01:05,973 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/c95173f4df9b4784a49552aa6fd47bf3 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/c95173f4df9b4784a49552aa6fd47bf3 2024-12-06T16:01:05,975 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/52eea21c14fd4bc9ad27b6c944029622 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/52eea21c14fd4bc9ad27b6c944029622 2024-12-06T16:01:05,976 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/0f827ce12fa64c96beadfb3595973869 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/0f827ce12fa64c96beadfb3595973869 2024-12-06T16:01:05,977 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/c0c937516075446085bd9d21c2cc7d0b to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/c0c937516075446085bd9d21c2cc7d0b 2024-12-06T16:01:05,978 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/8956e325d48f4f2691df11e757efc359 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/8956e325d48f4f2691df11e757efc359 2024-12-06T16:01:05,980 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/e8a0a42aeb224ecb826258eb878db948 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/e8a0a42aeb224ecb826258eb878db948 2024-12-06T16:01:05,981 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/40cfd0cc6c8f47809c2953b7b1e4ddc4 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/40cfd0cc6c8f47809c2953b7b1e4ddc4 2024-12-06T16:01:05,982 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/08c8742946864c9f9d6ac5f9ea3ed45f to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/08c8742946864c9f9d6ac5f9ea3ed45f 2024-12-06T16:01:05,983 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/09f6ad844e984d62a2e413307fc65cf8 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/09f6ad844e984d62a2e413307fc65cf8 2024-12-06T16:01:05,984 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/56ee5c2f5fc04f40a0b753b07fffaceb to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/56ee5c2f5fc04f40a0b753b07fffaceb 2024-12-06T16:01:05,985 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/14616e15b1a34ffbb816792eab8ddcc0 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/14616e15b1a34ffbb816792eab8ddcc0 2024-12-06T16:01:05,986 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/842b803c611e4ec5b4a0dbaf8679e63b to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/842b803c611e4ec5b4a0dbaf8679e63b 2024-12-06T16:01:05,987 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/92104cae7974403b8b492cd0007c0c73 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/92104cae7974403b8b492cd0007c0c73 2024-12-06T16:01:05,988 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/da8d4fafaaa84b6a9fc64bfaf41779f3 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/da8d4fafaaa84b6a9fc64bfaf41779f3 2024-12-06T16:01:05,989 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/bd2c3b8609fc4af689ac6866265fa3d4 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/bd2c3b8609fc4af689ac6866265fa3d4 2024-12-06T16:01:05,990 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/1c2c85f9127b4ecb95ceec2e599b6f41 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/1c2c85f9127b4ecb95ceec2e599b6f41 2024-12-06T16:01:05,991 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/42fe32e9d0024881bafcabf4d91f8e0b to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/42fe32e9d0024881bafcabf4d91f8e0b 2024-12-06T16:01:05,992 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/1d63aadd7d4743048773b6820ac28d5a to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/1d63aadd7d4743048773b6820ac28d5a 2024-12-06T16:01:05,993 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/022e94ec72014382a1d62f48bba40b03 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/022e94ec72014382a1d62f48bba40b03 2024-12-06T16:01:05,994 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/f4b0f41751164b72add566aae08bb760 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/f4b0f41751164b72add566aae08bb760 2024-12-06T16:01:05,996 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/a40c7e21c99d4b02aa7ae4e667d38cc7 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/a40c7e21c99d4b02aa7ae4e667d38cc7 2024-12-06T16:01:05,997 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/e56c148a815f42cdaa8d34e7beed587a to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/e56c148a815f42cdaa8d34e7beed587a 2024-12-06T16:01:05,998 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/c7d2168c0493421eb85012fdd10c1af3 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/c7d2168c0493421eb85012fdd10c1af3 2024-12-06T16:01:06,000 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/6cb91ab09bc84f228e839b9dba15f645 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/6cb91ab09bc84f228e839b9dba15f645 2024-12-06T16:01:06,004 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/173effb815db4c5db9a44d840b55fb5b to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/173effb815db4c5db9a44d840b55fb5b 2024-12-06T16:01:06,006 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/7b9600ffe7e24320bd4ecfb313efed87 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/7b9600ffe7e24320bd4ecfb313efed87 2024-12-06T16:01:06,008 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/dc57db6371ed4098b90e625265e3e1f8 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/dc57db6371ed4098b90e625265e3e1f8 2024-12-06T16:01:06,009 DEBUG [StoreCloser-TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/43a2a164ebf94fbcad793a1d63c1d8d5 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/43a2a164ebf94fbcad793a1d63c1d8d5 2024-12-06T16:01:06,015 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/recovered.edits/571.seqid, newMaxSeqId=571, maxSeqId=1 2024-12-06T16:01:06,016 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. 2024-12-06T16:01:06,017 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1676): Region close journal for 81bfe42898e92490dcd2dea3186de72d: Waiting for close lock at 1733500864516Running coprocessor pre-close hooks at 1733500864516Disabling compacts and flushes for region at 1733500864516Disabling writes for close at 1733500864516Obtaining lock to block concurrent updates at 1733500864516Preparing flush snapshotting stores in 81bfe42898e92490dcd2dea3186de72d at 1733500864516Finished memstore snapshotting TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1733500864517 (+1 ms)Flushing stores of TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d. at 1733500864518 (+1 ms)Flushing 81bfe42898e92490dcd2dea3186de72d/A: creating writer at 1733500864518Flushing 81bfe42898e92490dcd2dea3186de72d/A: appending metadata at 1733500864521 (+3 ms)Flushing 81bfe42898e92490dcd2dea3186de72d/A: closing flushed file at 1733500864521Flushing 81bfe42898e92490dcd2dea3186de72d/B: creating writer at 1733500864933 (+412 ms)Flushing 81bfe42898e92490dcd2dea3186de72d/B: appending metadata at 1733500864937 (+4 ms)Flushing 81bfe42898e92490dcd2dea3186de72d/B: closing flushed file at 1733500864937Flushing 81bfe42898e92490dcd2dea3186de72d/C: creating writer at 1733500865349 (+412 ms)Flushing 81bfe42898e92490dcd2dea3186de72d/C: appending metadata at 1733500865352 (+3 ms)Flushing 81bfe42898e92490dcd2dea3186de72d/C: closing flushed file at 1733500865352Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e533649: reopening flushed file at 1733500865762 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b6e5099: reopening flushed file at 1733500865767 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@776155a2: reopening flushed file at 1733500865773 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 81bfe42898e92490dcd2dea3186de72d in 1263ms, sequenceid=568, compaction requested=false at 1733500865779 (+6 ms)Writing region close event to WAL at 1733500866011 (+232 ms)Running coprocessor post-close hooks at 1733500866016 (+5 ms)Closed at 1733500866016 2024-12-06T16:01:06,019 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(157): Closed 81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:01:06,020 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=81bfe42898e92490dcd2dea3186de72d, regionState=CLOSED 2024-12-06T16:01:06,022 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; CloseRegionProcedure 81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 because future has completed 2024-12-06T16:01:06,025 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-06T16:01:06,025 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseRegionProcedure 81bfe42898e92490dcd2dea3186de72d, server=85bef17d9292,46105,1733500827732 in 1.6660 sec 2024-12-06T16:01:06,026 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-12-06T16:01:06,026 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81bfe42898e92490dcd2dea3186de72d, UNASSIGN in 1.6740 sec 2024-12-06T16:01:06,029 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-06T16:01:06,029 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6830 sec 2024-12-06T16:01:06,030 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733500866030"}]},"ts":"1733500866030"} 2024-12-06T16:01:06,033 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-06T16:01:06,033 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-06T16:01:06,035 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7000 sec 2024-12-06T16:01:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-06T16:01:06,482 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:01:06,485 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-06T16:01:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:01:06,491 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=25, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:01:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-06T16:01:06,492 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=25, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:01:06,498 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:01:06,503 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A, FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B, FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C, FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/recovered.edits] 2024-12-06T16:01:06,506 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/4f30d5a4f2744f4482971d4590fd6ff1 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/4f30d5a4f2744f4482971d4590fd6ff1 2024-12-06T16:01:06,508 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/90a3286cdbb44ce6a6dc9abfcf414bae to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/A/90a3286cdbb44ce6a6dc9abfcf414bae 2024-12-06T16:01:06,512 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/9f8bd252ebc04dda836c4e6d8d927f19 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/9f8bd252ebc04dda836c4e6d8d927f19 2024-12-06T16:01:06,513 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/e6dbb054ec7546ef820e8e76ff28b63c to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/B/e6dbb054ec7546ef820e8e76ff28b63c 2024-12-06T16:01:06,516 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/626f6cf14324443c9daa4b0ce894a00c to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/626f6cf14324443c9daa4b0ce894a00c 2024-12-06T16:01:06,517 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/d700897ec2374d449a78c71751e5d301 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/C/d700897ec2374d449a78c71751e5d301 2024-12-06T16:01:06,520 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/recovered.edits/571.seqid to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d/recovered.edits/571.seqid 2024-12-06T16:01:06,521 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81bfe42898e92490dcd2dea3186de72d 2024-12-06T16:01:06,521 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-06T16:01:06,528 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=25, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:01:06,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-06T16:01:06,539 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-06T16:01:06,576 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-06T16:01:06,578 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=25, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:01:06,578 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-06T16:01:06,579 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733500866578"}]},"ts":"9223372036854775807"} 2024-12-06T16:01:06,584 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-06T16:01:06,584 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 81bfe42898e92490dcd2dea3186de72d, NAME => 'TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d.', STARTKEY => '', ENDKEY => ''}] 2024-12-06T16:01:06,584 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-06T16:01:06,584 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733500866584"}]},"ts":"9223372036854775807"} 2024-12-06T16:01:06,588 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-06T16:01:06,589 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=25, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:01:06,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 103 msec 2024-12-06T16:01:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-06T16:01:06,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-06T16:01:06,606 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:01:06,619 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=237 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/85bef17d9292:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_130634295_22 at /127.0.0.1:59570 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;85bef17d9292:46105-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=494 (was 400) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8876 (was 9570) 2024-12-06T16:01:06,631 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=237, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=494, ProcessCount=12, AvailableMemoryMB=8875 2024-12-06T16:01:06,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-06T16:01:06,634 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T16:01:06,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-06T16:01:06,637 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T16:01:06,637 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:06,637 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 26 2024-12-06T16:01:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-06T16:01:06,638 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T16:01:06,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741960_1136 (size=959) 2024-12-06T16:01:06,654 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d 2024-12-06T16:01:06,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741961_1137 (size=53) 2024-12-06T16:01:06,663 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T16:01:06,664 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0d7db0f41a15490076613839fa83ecf9, disabling compactions & flushes 2024-12-06T16:01:06,664 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:06,664 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:06,664 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. after waiting 0 ms 2024-12-06T16:01:06,664 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:06,664 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:06,664 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0d7db0f41a15490076613839fa83ecf9: Waiting for close lock at 1733500866664Disabling compacts and flushes for region at 1733500866664Disabling writes for close at 1733500866664Writing region close event to WAL at 1733500866664Closed at 1733500866664 2024-12-06T16:01:06,665 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T16:01:06,666 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733500866665"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733500866665"}]},"ts":"1733500866665"} 2024-12-06T16:01:06,667 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T16:01:06,668 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T16:01:06,668 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733500866668"}]},"ts":"1733500866668"} 2024-12-06T16:01:06,670 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-06T16:01:06,670 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d7db0f41a15490076613839fa83ecf9, ASSIGN}] 2024-12-06T16:01:06,671 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d7db0f41a15490076613839fa83ecf9, ASSIGN 2024-12-06T16:01:06,673 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d7db0f41a15490076613839fa83ecf9, ASSIGN; state=OFFLINE, location=85bef17d9292,46105,1733500827732; forceNewPlan=false, retain=false 2024-12-06T16:01:06,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-06T16:01:06,823 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=0d7db0f41a15490076613839fa83ecf9, regionState=OPENING, regionLocation=85bef17d9292,46105,1733500827732 2024-12-06T16:01:06,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d7db0f41a15490076613839fa83ecf9, ASSIGN because future has completed 2024-12-06T16:01:06,828 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732}] 2024-12-06T16:01:06,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-06T16:01:06,984 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:06,984 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} 2024-12-06T16:01:06,985 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:06,985 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T16:01:06,985 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:06,985 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:06,987 INFO [StoreOpener-0d7db0f41a15490076613839fa83ecf9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:06,989 INFO [StoreOpener-0d7db0f41a15490076613839fa83ecf9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d7db0f41a15490076613839fa83ecf9 columnFamilyName A 2024-12-06T16:01:06,989 DEBUG [StoreOpener-0d7db0f41a15490076613839fa83ecf9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:06,990 INFO [StoreOpener-0d7db0f41a15490076613839fa83ecf9-1 {}] regionserver.HStore(327): Store=0d7db0f41a15490076613839fa83ecf9/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T16:01:06,990 INFO [StoreOpener-0d7db0f41a15490076613839fa83ecf9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:06,991 INFO [StoreOpener-0d7db0f41a15490076613839fa83ecf9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d7db0f41a15490076613839fa83ecf9 columnFamilyName B 2024-12-06T16:01:06,991 DEBUG [StoreOpener-0d7db0f41a15490076613839fa83ecf9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:06,992 INFO [StoreOpener-0d7db0f41a15490076613839fa83ecf9-1 {}] regionserver.HStore(327): Store=0d7db0f41a15490076613839fa83ecf9/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T16:01:06,992 INFO [StoreOpener-0d7db0f41a15490076613839fa83ecf9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:06,993 INFO [StoreOpener-0d7db0f41a15490076613839fa83ecf9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d7db0f41a15490076613839fa83ecf9 columnFamilyName C 2024-12-06T16:01:06,993 DEBUG [StoreOpener-0d7db0f41a15490076613839fa83ecf9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:06,993 INFO [StoreOpener-0d7db0f41a15490076613839fa83ecf9-1 {}] regionserver.HStore(327): Store=0d7db0f41a15490076613839fa83ecf9/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T16:01:06,993 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:06,994 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:06,994 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:06,995 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:06,995 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:06,996 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T16:01:06,997 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:07,000 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T16:01:07,001 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened 0d7db0f41a15490076613839fa83ecf9; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73793280, jitterRate=0.09960556030273438}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T16:01:07,001 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:07,003 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for 0d7db0f41a15490076613839fa83ecf9: Running coprocessor pre-open hook at 1733500866986Writing region info on filesystem at 1733500866986Initializing all the Stores at 1733500866986Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733500866987 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733500866987Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733500866987Cleaning up temporary data from old regions at 1733500866995 (+8 ms)Running coprocessor post-open hooks at 1733500867001 (+6 ms)Region opened successfully at 1733500867003 (+2 ms) 2024-12-06T16:01:07,004 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., pid=28, masterSystemTime=1733500866980 2024-12-06T16:01:07,006 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:07,006 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:07,007 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=0d7db0f41a15490076613839fa83ecf9, regionState=OPEN, openSeqNum=2, regionLocation=85bef17d9292,46105,1733500827732 2024-12-06T16:01:07,008 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 because future has completed 2024-12-06T16:01:07,012 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-06T16:01:07,012 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 in 182 msec 2024-12-06T16:01:07,017 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-06T16:01:07,017 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d7db0f41a15490076613839fa83ecf9, ASSIGN in 342 msec 2024-12-06T16:01:07,018 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T16:01:07,018 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733500867018"}]},"ts":"1733500867018"} 2024-12-06T16:01:07,021 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-06T16:01:07,022 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T16:01:07,023 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 388 msec 2024-12-06T16:01:07,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-06T16:01:07,263 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:01:07,264 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@705aba3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:07,264 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:01:07,264 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:01:07,266 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:01:07,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:01:07,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:01:07,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42e14557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:07,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:01:07,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:01:07,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:07,269 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:01:07,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@152d0274, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:07,270 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:01:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:01:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:01:07,273 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60208, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:01:07,274 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T16:01:07,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 85bef17d9292,45907,1733500826847 2024-12-06T16:01:07,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@14c737b8 2024-12-06T16:01:07,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T16:01:07,276 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50724, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T16:01:07,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-06T16:01:07,282 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T16:01:07,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-06T16:01:07,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741962_1138 (size=995) 2024-12-06T16:01:07,702 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-12-06T16:01:07,702 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-12-06T16:01:07,706 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-06T16:01:07,712 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d7db0f41a15490076613839fa83ecf9, REOPEN/MOVE}] 2024-12-06T16:01:07,713 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d7db0f41a15490076613839fa83ecf9, REOPEN/MOVE 2024-12-06T16:01:07,714 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=0d7db0f41a15490076613839fa83ecf9, regionState=CLOSING, regionLocation=85bef17d9292,46105,1733500827732 2024-12-06T16:01:07,716 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d7db0f41a15490076613839fa83ecf9, REOPEN/MOVE because future has completed 2024-12-06T16:01:07,717 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-06T16:01:07,717 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732}] 2024-12-06T16:01:07,870 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:07,870 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-06T16:01:07,870 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 0d7db0f41a15490076613839fa83ecf9, disabling compactions & flushes 2024-12-06T16:01:07,870 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:07,870 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:07,870 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. after waiting 0 ms 2024-12-06T16:01:07,870 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:07,874 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-06T16:01:07,875 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:07,875 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 0d7db0f41a15490076613839fa83ecf9: Waiting for close lock at 1733500867870Running coprocessor pre-close hooks at 1733500867870Disabling compacts and flushes for region at 1733500867870Disabling writes for close at 1733500867870Writing region close event to WAL at 1733500867870Running coprocessor post-close hooks at 1733500867875 (+5 ms)Closed at 1733500867875 2024-12-06T16:01:07,875 WARN [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionServer(3299): Not adding moved region record: 0d7db0f41a15490076613839fa83ecf9 to self. 2024-12-06T16:01:07,877 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:07,878 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=0d7db0f41a15490076613839fa83ecf9, regionState=CLOSED 2024-12-06T16:01:07,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 because future has completed 2024-12-06T16:01:07,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-06T16:01:07,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 in 163 msec 2024-12-06T16:01:07,883 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d7db0f41a15490076613839fa83ecf9, REOPEN/MOVE; state=CLOSED, location=85bef17d9292,46105,1733500827732; forceNewPlan=false, retain=true 2024-12-06T16:01:08,034 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=0d7db0f41a15490076613839fa83ecf9, regionState=OPENING, regionLocation=85bef17d9292,46105,1733500827732 2024-12-06T16:01:08,036 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d7db0f41a15490076613839fa83ecf9, REOPEN/MOVE because future has completed 2024-12-06T16:01:08,037 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732}] 2024-12-06T16:01:08,193 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:08,193 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} 2024-12-06T16:01:08,193 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:08,193 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T16:01:08,194 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:08,194 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:08,196 INFO [StoreOpener-0d7db0f41a15490076613839fa83ecf9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:08,202 INFO [StoreOpener-0d7db0f41a15490076613839fa83ecf9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d7db0f41a15490076613839fa83ecf9 columnFamilyName A 2024-12-06T16:01:08,204 DEBUG [StoreOpener-0d7db0f41a15490076613839fa83ecf9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:08,204 INFO [StoreOpener-0d7db0f41a15490076613839fa83ecf9-1 {}] regionserver.HStore(327): Store=0d7db0f41a15490076613839fa83ecf9/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T16:01:08,205 INFO [StoreOpener-0d7db0f41a15490076613839fa83ecf9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:08,205 INFO [StoreOpener-0d7db0f41a15490076613839fa83ecf9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d7db0f41a15490076613839fa83ecf9 columnFamilyName B 2024-12-06T16:01:08,205 DEBUG [StoreOpener-0d7db0f41a15490076613839fa83ecf9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:08,206 INFO [StoreOpener-0d7db0f41a15490076613839fa83ecf9-1 {}] regionserver.HStore(327): Store=0d7db0f41a15490076613839fa83ecf9/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T16:01:08,206 INFO [StoreOpener-0d7db0f41a15490076613839fa83ecf9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:08,207 INFO [StoreOpener-0d7db0f41a15490076613839fa83ecf9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d7db0f41a15490076613839fa83ecf9 columnFamilyName C 2024-12-06T16:01:08,207 DEBUG [StoreOpener-0d7db0f41a15490076613839fa83ecf9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:08,207 INFO [StoreOpener-0d7db0f41a15490076613839fa83ecf9-1 {}] regionserver.HStore(327): Store=0d7db0f41a15490076613839fa83ecf9/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T16:01:08,207 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:08,208 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:08,209 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:08,210 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:08,210 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:08,210 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T16:01:08,212 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:08,213 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 0d7db0f41a15490076613839fa83ecf9; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59738974, jitterRate=-0.1098199188709259}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T16:01:08,213 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:08,214 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 0d7db0f41a15490076613839fa83ecf9: Running coprocessor pre-open hook at 1733500868194Writing region info on filesystem at 1733500868194Initializing all the Stores at 1733500868195 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733500868195Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733500868195Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733500868195Cleaning up temporary data from old regions at 1733500868210 (+15 ms)Running coprocessor post-open hooks at 1733500868213 (+3 ms)Region opened successfully at 1733500868214 (+1 ms) 2024-12-06T16:01:08,215 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., pid=33, masterSystemTime=1733500868189 2024-12-06T16:01:08,217 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:08,217 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:08,218 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=0d7db0f41a15490076613839fa83ecf9, regionState=OPEN, openSeqNum=5, regionLocation=85bef17d9292,46105,1733500827732 2024-12-06T16:01:08,220 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 because future has completed 2024-12-06T16:01:08,222 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=31 2024-12-06T16:01:08,222 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 in 184 msec 2024-12-06T16:01:08,225 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-06T16:01:08,225 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d7db0f41a15490076613839fa83ecf9, REOPEN/MOVE in 511 msec 2024-12-06T16:01:08,229 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-06T16:01:08,229 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 522 msec 2024-12-06T16:01:08,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 947 msec 2024-12-06T16:01:08,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-06T16:01:08,236 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:01:08,237 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13460079, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:08,237 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:01:08,237 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:01:08,238 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:01:08,238 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:01:08,238 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:01:08,239 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22a1f764, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:08,239 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:01:08,239 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:01:08,239 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:08,240 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50740, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:01:08,241 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28d1de7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:08,242 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a297d33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:08,242 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:01:08,242 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:01:08,252 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:01:08,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:01:08,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:01:08,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32bb8111, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:08,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:01:08,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:01:08,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:08,254 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:01:08,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b5122a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:08,256 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21bc0752, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:08,256 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:01:08,256 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:01:08,257 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:01:08,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:01:08,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:01:08,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e72cde8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:08,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:01:08,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:01:08,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:08,259 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50784, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:01:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d644090, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:08,261 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c7c85e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:08,261 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:01:08,261 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:01:08,262 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:01:08,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:01:08,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:01:08,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@124a4c29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:08,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:01:08,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:01:08,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:08,264 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50796, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:01:08,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f2edcc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:08,265 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c0ab7ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:08,265 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:01:08,265 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:01:08,266 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:01:08,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:01:08,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:01:08,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8c028b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:08,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:01:08,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:01:08,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:08,268 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:01:08,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d2a598, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:08,269 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1987d108, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:08,269 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:01:08,269 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:01:08,270 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:01:08,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:01:08,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:01:08,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7631b0db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:08,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:01:08,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:01:08,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:08,272 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50822, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:01:08,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69354af0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:08,273 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e5fd39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:08,273 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:01:08,273 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:01:08,274 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:01:08,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:01:08,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:01:08,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e595508, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:08,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:01:08,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:01:08,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:08,276 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50838, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:01:08,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18847951, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:08,277 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18690ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:08,277 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:01:08,277 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:01:08,277 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:01:08,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:01:08,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:01:08,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4662add3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:08,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:01:08,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:01:08,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:08,279 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50864, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:01:08,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c6c65e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:08,280 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41e58620, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:08,280 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:01:08,280 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:01:08,281 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:01:08,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:01:08,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:01:08,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62f22372, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:08,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:01:08,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:01:08,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:08,282 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50880, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:01:08,282 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9f9fc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:08,283 DEBUG [Thread-694 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:01:08,283 DEBUG [Thread-686 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:01:08,284 DEBUG [Thread-703 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:01:08,284 DEBUG [Thread-697 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:01:08,284 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:01:08,284 DEBUG [Thread-688 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:01:08,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:01:08,285 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:01:08,285 DEBUG [Thread-701 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:01:08,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:01:08,285 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:01:08,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:01:08,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:01:08,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:01:08,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:01:08,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:01:08,286 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:01:08,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees 2024-12-06T16:01:08,286 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:01:08,286 DEBUG [Thread-688 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:01:08,287 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60222, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:01:08,288 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:01:08,288 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:01:08,287 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60238, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:01:08,288 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:01:08,288 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60252, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:01:08,289 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:01:08,290 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:01:08,290 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:01:08,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5] 2024-12-06T16:01:08,291 DEBUG [Thread-692 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:01:08,292 DEBUG [Thread-699 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:01:08,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5] 2024-12-06T16:01:08,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-06T16:01:08,293 DEBUG [Thread-690 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:01:08,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:01:08,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5] 2024-12-06T16:01:08,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:01:08,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5] 2024-12-06T16:01:08,296 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60272, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:01:08,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5] 2024-12-06T16:01:08,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:01:08,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:01:08,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5] 2024-12-06T16:01:08,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5] 2024-12-06T16:01:08,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:01:08,300 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:01:08,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:01:08,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5] 2024-12-06T16:01:08,306 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60290, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:01:08,310 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:01:08,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5] 2024-12-06T16:01:08,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d7db0f41a15490076613839fa83ecf9 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-06T16:01:08,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:08,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:08,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500928366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:08,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:08,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60246 deadline: 1733500928367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:08,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206cb9cd903a96a435bb6163f5d73fe5473_0d7db0f41a15490076613839fa83ecf9 is 50, key is test_row_0/A:col10/1733500868306/Put/seqid=0 2024-12-06T16:01:08,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:08,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500928368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:08,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:08,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:08,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:08,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:08,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:08,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:08,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:08,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:08,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:08,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-06T16:01:08,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:08,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733500928388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:08,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:08,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500928392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:08,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:08,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:08,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:08,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:08,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:08,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:08,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741963_1139 (size=17034) 2024-12-06T16:01:08,442 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:08,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-06T16:01:08,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:08,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:08,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:08,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:08,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:08,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:08,457 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206cb9cd903a96a435bb6163f5d73fe5473_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206cb9cd903a96a435bb6163f5d73fe5473_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:08,458 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/c0b2e11112104a1a89d88e7d79cf6caf, store: [table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:08,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/c0b2e11112104a1a89d88e7d79cf6caf is 175, key is test_row_0/A:col10/1733500868306/Put/seqid=0 2024-12-06T16:01:08,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:08,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60246 deadline: 1733500928502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:08,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:08,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:08,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:08,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:08,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500928512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:08,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741964_1140 (size=48139) 2024-12-06T16:01:08,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:08,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:08,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:08,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:08,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733500928513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:08,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:08,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:08,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:08,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:08,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500928513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:08,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:08,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500928513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:08,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:08,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:08,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:08,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-06T16:01:08,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:08,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:08,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:08,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:08,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:08,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:08,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-06T16:01:08,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:08,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60246 deadline: 1733500928713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:08,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:08,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:08,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:08,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:08,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500928722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:08,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:08,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:08,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733500928722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:08,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500928722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:08,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:08,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:08,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:08,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:08,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500928733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:08,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:08,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:08,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:08,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-06T16:01:08,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:08,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:08,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:08,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:08,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:08,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:08,919 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/c0b2e11112104a1a89d88e7d79cf6caf 2024-12-06T16:01:08,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-06T16:01:08,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:08,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:08,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:08,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:08,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-06T16:01:08,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/d8891743d2f740ed97997b54eb9bea03 is 50, key is test_row_0/B:col10/1733500868306/Put/seqid=0 2024-12-06T16:01:09,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741965_1141 (size=12001) 2024-12-06T16:01:09,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60246 deadline: 1733500929032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:09,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:09,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500929032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:09,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:09,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733500929043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:09,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:09,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500929043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:09,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:09,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:09,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500929043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:09,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:09,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-06T16:01:09,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:09,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:09,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:09,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:09,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:09,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-06T16:01:09,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:09,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:09,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:09,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:09,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:09,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-06T16:01:09,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:09,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:09,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:09,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:09,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:09,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/d8891743d2f740ed97997b54eb9bea03 2024-12-06T16:01:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-06T16:01:09,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/c5476b0259854be9864031ac442a32d2 is 50, key is test_row_0/C:col10/1733500868306/Put/seqid=0 2024-12-06T16:01:09,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741966_1142 (size=12001) 2024-12-06T16:01:09,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/c5476b0259854be9864031ac442a32d2 2024-12-06T16:01:09,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/c0b2e11112104a1a89d88e7d79cf6caf as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/c0b2e11112104a1a89d88e7d79cf6caf 2024-12-06T16:01:09,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/c0b2e11112104a1a89d88e7d79cf6caf, entries=250, sequenceid=16, filesize=47.0 K 2024-12-06T16:01:09,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/d8891743d2f740ed97997b54eb9bea03 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/d8891743d2f740ed97997b54eb9bea03 2024-12-06T16:01:09,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/d8891743d2f740ed97997b54eb9bea03, entries=150, sequenceid=16, filesize=11.7 K 2024-12-06T16:01:09,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/c5476b0259854be9864031ac442a32d2 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/c5476b0259854be9864031ac442a32d2 2024-12-06T16:01:09,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/c5476b0259854be9864031ac442a32d2, entries=150, sequenceid=16, filesize=11.7 K 2024-12-06T16:01:09,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 0d7db0f41a15490076613839fa83ecf9 in 1221ms, sequenceid=16, compaction requested=false 2024-12-06T16:01:09,531 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-06T16:01:09,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:09,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-06T16:01:09,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:09,547 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing 0d7db0f41a15490076613839fa83ecf9 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-06T16:01:09,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:09,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:09,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412068f04fb006e2c44f9b3b902a8d44be608_0d7db0f41a15490076613839fa83ecf9 is 50, key is test_row_0/A:col10/1733500868363/Put/seqid=0 2024-12-06T16:01:09,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:09,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500929584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:09,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:09,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:09,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60246 deadline: 1733500929607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:09,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:09,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500929610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:09,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:09,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:09,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500929616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:09,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:09,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:09,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733500929618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:09,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:09,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741967_1143 (size=12154) 2024-12-06T16:01:09,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:09,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500929723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:09,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:09,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60246 deadline: 1733500929732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:09,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500929733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:09,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:09,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:09,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733500929734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:09,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:09,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:09,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:09,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500929734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:09,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:09,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:09,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500929931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:09,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:09,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:09,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60246 deadline: 1733500929952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:09,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:09,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500929952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:09,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:09,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:09,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:09,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733500929952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:09,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:09,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500929952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:09,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:09,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:09,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:10,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:10,046 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412068f04fb006e2c44f9b3b902a8d44be608_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412068f04fb006e2c44f9b3b902a8d44be608_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:10,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/c86903ceafdf479e828a8ae7a77acf3f, store: [table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:10,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/c86903ceafdf479e828a8ae7a77acf3f is 175, key is test_row_0/A:col10/1733500868363/Put/seqid=0 2024-12-06T16:01:10,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741968_1144 (size=30955) 2024-12-06T16:01:10,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:10,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500930242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:10,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:10,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:10,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:10,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:10,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60246 deadline: 1733500930262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:10,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:10,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733500930262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:10,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:10,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:10,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:10,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:10,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:10,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:10,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500930272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:10,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:10,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:10,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:10,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500930273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:10,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:10,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:10,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:10,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-06T16:01:10,462 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/c86903ceafdf479e828a8ae7a77acf3f 2024-12-06T16:01:10,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/15d3261d28684bdebe3a33e7f9f42fc0 is 50, key is test_row_0/B:col10/1733500868363/Put/seqid=0 2024-12-06T16:01:10,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741969_1145 (size=12001) 2024-12-06T16:01:10,504 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/15d3261d28684bdebe3a33e7f9f42fc0 2024-12-06T16:01:10,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/f8e5cae6116447868200e9464483e999 is 50, key is test_row_0/C:col10/1733500868363/Put/seqid=0 2024-12-06T16:01:10,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741970_1146 (size=12001) 2024-12-06T16:01:10,572 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/f8e5cae6116447868200e9464483e999 2024-12-06T16:01:10,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/c86903ceafdf479e828a8ae7a77acf3f as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/c86903ceafdf479e828a8ae7a77acf3f 2024-12-06T16:01:10,584 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/c86903ceafdf479e828a8ae7a77acf3f, entries=150, sequenceid=41, filesize=30.2 K 2024-12-06T16:01:10,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/15d3261d28684bdebe3a33e7f9f42fc0 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/15d3261d28684bdebe3a33e7f9f42fc0 2024-12-06T16:01:10,609 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/15d3261d28684bdebe3a33e7f9f42fc0, entries=150, sequenceid=41, filesize=11.7 K 2024-12-06T16:01:10,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/f8e5cae6116447868200e9464483e999 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/f8e5cae6116447868200e9464483e999 2024-12-06T16:01:10,620 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/f8e5cae6116447868200e9464483e999, entries=150, sequenceid=41, filesize=11.7 K 2024-12-06T16:01:10,621 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 0d7db0f41a15490076613839fa83ecf9 in 1074ms, sequenceid=41, compaction requested=false 2024-12-06T16:01:10,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:10,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:10,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-12-06T16:01:10,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-12-06T16:01:10,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-12-06T16:01:10,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3330 sec 2024-12-06T16:01:10,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees in 2.3510 sec 2024-12-06T16:01:10,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:10,755 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:01:10,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d7db0f41a15490076613839fa83ecf9 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-06T16:01:10,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206ddf4e89a12294b12a07275a9d35bcb12_0d7db0f41a15490076613839fa83ecf9 is 50, key is test_row_0/A:col10/1733500869576/Put/seqid=0 2024-12-06T16:01:10,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741971_1147 (size=14594) 2024-12-06T16:01:10,834 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:10,840 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206ddf4e89a12294b12a07275a9d35bcb12_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ddf4e89a12294b12a07275a9d35bcb12_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:10,842 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/7acc16ea6dc7428ba361ca5b29b3c9d3, store: [table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:10,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/7acc16ea6dc7428ba361ca5b29b3c9d3 is 175, key is test_row_0/A:col10/1733500869576/Put/seqid=0 2024-12-06T16:01:10,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:10,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60246 deadline: 1733500930845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:10,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:10,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500930847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:10,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:10,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:10,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:10,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:10,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:10,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:10,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:10,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733500930851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:10,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:10,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:10,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:10,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:10,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500930852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:10,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:10,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:10,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:10,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:10,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500930856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:10,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:10,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:10,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:10,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741972_1148 (size=39549) 2024-12-06T16:01:10,885 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/7acc16ea6dc7428ba361ca5b29b3c9d3 2024-12-06T16:01:10,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/10858766e5fd444e8f1cee1af0c16530 is 50, key is test_row_0/B:col10/1733500869576/Put/seqid=0 2024-12-06T16:01:10,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741973_1149 (size=12001) 2024-12-06T16:01:10,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:10,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:10,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60246 deadline: 1733500930963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:10,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500930963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:10,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:10,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:10,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:10,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:10,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733500930964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:10,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:10,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500930964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:10,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:10,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:10,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:10,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:10,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500930972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:10,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:10,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:10,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:11,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:11,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60246 deadline: 1733500931171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:11,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:11,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500931171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:11,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:11,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:11,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733500931172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:11,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:11,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:11,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500931172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:11,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:11,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:11,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500931182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:11,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:11,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/10858766e5fd444e8f1cee1af0c16530 2024-12-06T16:01:11,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/e086a6cb71f84013841049eb5a4beaff is 50, key is test_row_0/C:col10/1733500869576/Put/seqid=0 2024-12-06T16:01:11,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741974_1150 (size=12001) 2024-12-06T16:01:11,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/e086a6cb71f84013841049eb5a4beaff 2024-12-06T16:01:11,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/7acc16ea6dc7428ba361ca5b29b3c9d3 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/7acc16ea6dc7428ba361ca5b29b3c9d3 2024-12-06T16:01:11,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/7acc16ea6dc7428ba361ca5b29b3c9d3, entries=200, sequenceid=54, filesize=38.6 K 2024-12-06T16:01:11,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/10858766e5fd444e8f1cee1af0c16530 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/10858766e5fd444e8f1cee1af0c16530 2024-12-06T16:01:11,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/10858766e5fd444e8f1cee1af0c16530, entries=150, sequenceid=54, filesize=11.7 K 2024-12-06T16:01:11,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/e086a6cb71f84013841049eb5a4beaff as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/e086a6cb71f84013841049eb5a4beaff 2024-12-06T16:01:11,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/e086a6cb71f84013841049eb5a4beaff, entries=150, sequenceid=54, filesize=11.7 K 2024-12-06T16:01:11,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 0d7db0f41a15490076613839fa83ecf9 in 647ms, sequenceid=54, compaction requested=true 2024-12-06T16:01:11,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:11,402 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:11,403 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:11,403 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 0d7db0f41a15490076613839fa83ecf9/A is initiating minor compaction (all files) 2024-12-06T16:01:11,404 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d7db0f41a15490076613839fa83ecf9/A in TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:11,404 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/c0b2e11112104a1a89d88e7d79cf6caf, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/c86903ceafdf479e828a8ae7a77acf3f, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/7acc16ea6dc7428ba361ca5b29b3c9d3] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp, totalSize=115.9 K 2024-12-06T16:01:11,404 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:11,404 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. files: [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/c0b2e11112104a1a89d88e7d79cf6caf, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/c86903ceafdf479e828a8ae7a77acf3f, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/7acc16ea6dc7428ba361ca5b29b3c9d3] 2024-12-06T16:01:11,405 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0b2e11112104a1a89d88e7d79cf6caf, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733500868297 2024-12-06T16:01:11,405 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting c86903ceafdf479e828a8ae7a77acf3f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1733500868363 2024-12-06T16:01:11,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d7db0f41a15490076613839fa83ecf9:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:01:11,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:11,405 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:11,406 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7acc16ea6dc7428ba361ca5b29b3c9d3, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733500869576 2024-12-06T16:01:11,407 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:11,407 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 0d7db0f41a15490076613839fa83ecf9/B is initiating minor compaction (all files) 2024-12-06T16:01:11,407 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d7db0f41a15490076613839fa83ecf9/B in TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:11,408 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/d8891743d2f740ed97997b54eb9bea03, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/15d3261d28684bdebe3a33e7f9f42fc0, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/10858766e5fd444e8f1cee1af0c16530] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp, totalSize=35.2 K 2024-12-06T16:01:11,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d7db0f41a15490076613839fa83ecf9:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:01:11,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:11,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d7db0f41a15490076613839fa83ecf9:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:01:11,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:11,408 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting d8891743d2f740ed97997b54eb9bea03, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733500868306 2024-12-06T16:01:11,409 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 15d3261d28684bdebe3a33e7f9f42fc0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1733500868363 2024-12-06T16:01:11,409 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 10858766e5fd444e8f1cee1af0c16530, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733500869576 2024-12-06T16:01:11,426 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:11,430 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d7db0f41a15490076613839fa83ecf9#B#compaction#133 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:11,431 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/8f8d004ea376416ea789e469ad61acc7 is 50, key is test_row_0/B:col10/1733500869576/Put/seqid=0 2024-12-06T16:01:11,438 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206fee57314169e4a67afce2b5ae24c37d4_0d7db0f41a15490076613839fa83ecf9 store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:11,444 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206fee57314169e4a67afce2b5ae24c37d4_0d7db0f41a15490076613839fa83ecf9, store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:11,445 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206fee57314169e4a67afce2b5ae24c37d4_0d7db0f41a15490076613839fa83ecf9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:11,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741975_1151 (size=12104) 2024-12-06T16:01:11,465 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/8f8d004ea376416ea789e469ad61acc7 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/8f8d004ea376416ea789e469ad61acc7 2024-12-06T16:01:11,472 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d7db0f41a15490076613839fa83ecf9/B of 0d7db0f41a15490076613839fa83ecf9 into 8f8d004ea376416ea789e469ad61acc7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:11,472 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:11,472 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., storeName=0d7db0f41a15490076613839fa83ecf9/B, priority=13, startTime=1733500871405; duration=0sec 2024-12-06T16:01:11,472 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:11,472 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d7db0f41a15490076613839fa83ecf9:B 2024-12-06T16:01:11,472 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:11,473 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:11,474 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 0d7db0f41a15490076613839fa83ecf9/C is initiating minor compaction (all files) 2024-12-06T16:01:11,474 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d7db0f41a15490076613839fa83ecf9/C in TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:11,474 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/c5476b0259854be9864031ac442a32d2, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/f8e5cae6116447868200e9464483e999, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/e086a6cb71f84013841049eb5a4beaff] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp, totalSize=35.2 K 2024-12-06T16:01:11,474 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting c5476b0259854be9864031ac442a32d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733500868306 2024-12-06T16:01:11,475 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting f8e5cae6116447868200e9464483e999, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1733500868363 2024-12-06T16:01:11,476 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting e086a6cb71f84013841049eb5a4beaff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733500869576 2024-12-06T16:01:11,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741976_1152 (size=4469) 2024-12-06T16:01:11,490 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d7db0f41a15490076613839fa83ecf9#A#compaction#132 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:11,491 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/a7d54b9ca03c4f8cb26d43afdb992010 is 175, key is test_row_0/A:col10/1733500869576/Put/seqid=0 2024-12-06T16:01:11,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:11,494 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:01:11,494 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d7db0f41a15490076613839fa83ecf9 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-06T16:01:11,506 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d7db0f41a15490076613839fa83ecf9#C#compaction#134 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:11,507 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/52557655d8aa4afa82e74c90821660ae is 50, key is test_row_0/C:col10/1733500869576/Put/seqid=0 2024-12-06T16:01:11,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:11,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500931501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:11,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:11,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500931502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:11,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:11,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:11,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:11,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733500931504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:11,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:11,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:11,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60246 deadline: 1733500931508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:11,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:11,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500931509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:11,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:11,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:11,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741977_1153 (size=31058) 2024-12-06T16:01:11,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206bb3f6e75825b4d23872fd89fec8741e7_0d7db0f41a15490076613839fa83ecf9 is 50, key is test_row_0/A:col10/1733500871493/Put/seqid=0 2024-12-06T16:01:11,575 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/a7d54b9ca03c4f8cb26d43afdb992010 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/a7d54b9ca03c4f8cb26d43afdb992010 2024-12-06T16:01:11,577 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T16:01:11,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741978_1154 (size=12104) 2024-12-06T16:01:11,589 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d7db0f41a15490076613839fa83ecf9/A of 0d7db0f41a15490076613839fa83ecf9 into a7d54b9ca03c4f8cb26d43afdb992010(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:11,589 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:11,589 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., storeName=0d7db0f41a15490076613839fa83ecf9/A, priority=13, startTime=1733500871402; duration=0sec 2024-12-06T16:01:11,589 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:11,589 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d7db0f41a15490076613839fa83ecf9:A 2024-12-06T16:01:11,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741979_1155 (size=14594) 2024-12-06T16:01:11,591 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:11,595 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/52557655d8aa4afa82e74c90821660ae as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/52557655d8aa4afa82e74c90821660ae 2024-12-06T16:01:11,609 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206bb3f6e75825b4d23872fd89fec8741e7_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206bb3f6e75825b4d23872fd89fec8741e7_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:11,614 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d7db0f41a15490076613839fa83ecf9/C of 0d7db0f41a15490076613839fa83ecf9 into 52557655d8aa4afa82e74c90821660ae(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:11,614 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:11,614 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., storeName=0d7db0f41a15490076613839fa83ecf9/C, priority=13, startTime=1733500871408; duration=0sec 2024-12-06T16:01:11,614 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:11,614 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d7db0f41a15490076613839fa83ecf9:C 2024-12-06T16:01:11,617 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/784ee92929cb4f199c969cb1c24027a2, store: [table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:11,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/784ee92929cb4f199c969cb1c24027a2 is 175, key is test_row_0/A:col10/1733500871493/Put/seqid=0 2024-12-06T16:01:11,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:11,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500931621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:11,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:11,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60246 deadline: 1733500931622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:11,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:11,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:11,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500931622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:11,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:11,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:11,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500931622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:11,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:11,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:11,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733500931623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:11,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:11,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:11,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741980_1156 (size=39549) 2024-12-06T16:01:11,665 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/784ee92929cb4f199c969cb1c24027a2 2024-12-06T16:01:11,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/fd118a25ced4453ca94bf403306d550d is 50, key is test_row_0/B:col10/1733500871493/Put/seqid=0 2024-12-06T16:01:11,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741981_1157 (size=12001) 2024-12-06T16:01:11,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/fd118a25ced4453ca94bf403306d550d 2024-12-06T16:01:11,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/57da6b55f1fb42e8a1c9cca47303615b is 50, key is test_row_0/C:col10/1733500871493/Put/seqid=0 2024-12-06T16:01:11,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741982_1158 (size=12001) 2024-12-06T16:01:11,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/57da6b55f1fb42e8a1c9cca47303615b 2024-12-06T16:01:11,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/784ee92929cb4f199c969cb1c24027a2 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/784ee92929cb4f199c969cb1c24027a2 2024-12-06T16:01:11,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/784ee92929cb4f199c969cb1c24027a2, entries=200, sequenceid=82, filesize=38.6 K 2024-12-06T16:01:11,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/fd118a25ced4453ca94bf403306d550d as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/fd118a25ced4453ca94bf403306d550d 2024-12-06T16:01:11,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/fd118a25ced4453ca94bf403306d550d, entries=150, sequenceid=82, filesize=11.7 K 2024-12-06T16:01:11,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/57da6b55f1fb42e8a1c9cca47303615b as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/57da6b55f1fb42e8a1c9cca47303615b 2024-12-06T16:01:11,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/57da6b55f1fb42e8a1c9cca47303615b, entries=150, sequenceid=82, filesize=11.7 K 2024-12-06T16:01:11,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 0d7db0f41a15490076613839fa83ecf9 in 303ms, sequenceid=82, compaction requested=false 2024-12-06T16:01:11,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:11,839 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:01:11,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d7db0f41a15490076613839fa83ecf9 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-06T16:01:11,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:11,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206d7274579658a40a9bc3304137d8972b6_0d7db0f41a15490076613839fa83ecf9 is 50, key is test_row_0/A:col10/1733500871837/Put/seqid=0 2024-12-06T16:01:11,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500931876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:11,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:11,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500931880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:11,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:11,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60246 deadline: 1733500931884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:11,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:11,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500931884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:11,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:11,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:11,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:11,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733500931886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:11,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:11,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741983_1159 (size=14594) 2024-12-06T16:01:11,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:11,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500931992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:11,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:11,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:11,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500931992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:11,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:11,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:12,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:12,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500932004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:12,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:12,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:12,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60246 deadline: 1733500932004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:12,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:12,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:12,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733500932004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:12,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:12,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:12,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500932202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:12,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:12,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500932202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:12,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:12,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:12,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:12,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500932212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:12,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:12,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:12,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733500932222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:12,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:12,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60246 deadline: 1733500932222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:12,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:12,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:12,324 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:12,331 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206d7274579658a40a9bc3304137d8972b6_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206d7274579658a40a9bc3304137d8972b6_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:12,333 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/408e769fa08c439ba5b8dbc1ffcc435c, store: [table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:12,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/408e769fa08c439ba5b8dbc1ffcc435c is 175, key is test_row_0/A:col10/1733500871837/Put/seqid=0 2024-12-06T16:01:12,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741984_1160 (size=39549) 2024-12-06T16:01:12,352 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/408e769fa08c439ba5b8dbc1ffcc435c 2024-12-06T16:01:12,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/432e292fc1e14b859acfe24780f3a518 is 50, key is test_row_0/B:col10/1733500871837/Put/seqid=0 2024-12-06T16:01:12,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741985_1161 (size=12001) 2024-12-06T16:01:12,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/432e292fc1e14b859acfe24780f3a518 2024-12-06T16:01:12,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/94b9cfe17d574644b9ea90313d728233 is 50, key is test_row_0/C:col10/1733500871837/Put/seqid=0 2024-12-06T16:01:12,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741986_1162 (size=12001) 2024-12-06T16:01:12,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/94b9cfe17d574644b9ea90313d728233 2024-12-06T16:01:12,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/408e769fa08c439ba5b8dbc1ffcc435c as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/408e769fa08c439ba5b8dbc1ffcc435c 2024-12-06T16:01:12,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/408e769fa08c439ba5b8dbc1ffcc435c, entries=200, sequenceid=97, filesize=38.6 K 2024-12-06T16:01:12,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/432e292fc1e14b859acfe24780f3a518 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/432e292fc1e14b859acfe24780f3a518 2024-12-06T16:01:12,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/432e292fc1e14b859acfe24780f3a518, entries=150, sequenceid=97, filesize=11.7 K 2024-12-06T16:01:12,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/94b9cfe17d574644b9ea90313d728233 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/94b9cfe17d574644b9ea90313d728233 2024-12-06T16:01:12,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/94b9cfe17d574644b9ea90313d728233, entries=150, sequenceid=97, filesize=11.7 K 2024-12-06T16:01:12,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 0d7db0f41a15490076613839fa83ecf9 in 593ms, sequenceid=97, compaction requested=true 2024-12-06T16:01:12,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:12,432 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:12,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d7db0f41a15490076613839fa83ecf9:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:01:12,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:12,433 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:12,433 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110156 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:12,434 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 0d7db0f41a15490076613839fa83ecf9/A is initiating minor compaction (all files) 2024-12-06T16:01:12,434 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d7db0f41a15490076613839fa83ecf9/A in TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:12,434 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/a7d54b9ca03c4f8cb26d43afdb992010, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/784ee92929cb4f199c969cb1c24027a2, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/408e769fa08c439ba5b8dbc1ffcc435c] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp, totalSize=107.6 K 2024-12-06T16:01:12,434 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:12,434 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. files: [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/a7d54b9ca03c4f8cb26d43afdb992010, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/784ee92929cb4f199c969cb1c24027a2, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/408e769fa08c439ba5b8dbc1ffcc435c] 2024-12-06T16:01:12,435 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7d54b9ca03c4f8cb26d43afdb992010, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733500869576 2024-12-06T16:01:12,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d7db0f41a15490076613839fa83ecf9:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:01:12,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:12,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d7db0f41a15490076613839fa83ecf9:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:01:12,436 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 784ee92929cb4f199c969cb1c24027a2, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733500871487 2024-12-06T16:01:12,436 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:12,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:12,436 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 0d7db0f41a15490076613839fa83ecf9/B is initiating minor compaction (all files) 2024-12-06T16:01:12,436 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d7db0f41a15490076613839fa83ecf9/B in TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:12,436 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/8f8d004ea376416ea789e469ad61acc7, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/fd118a25ced4453ca94bf403306d550d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/432e292fc1e14b859acfe24780f3a518] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp, totalSize=35.3 K 2024-12-06T16:01:12,436 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 408e769fa08c439ba5b8dbc1ffcc435c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733500871499 2024-12-06T16:01:12,437 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f8d004ea376416ea789e469ad61acc7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733500869576 2024-12-06T16:01:12,437 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting fd118a25ced4453ca94bf403306d550d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733500871487 2024-12-06T16:01:12,437 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 432e292fc1e14b859acfe24780f3a518, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733500871507 2024-12-06T16:01:12,444 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:12,457 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d7db0f41a15490076613839fa83ecf9#B#compaction#142 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:12,458 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/4067755f0a174bf7987fe9d23a8d2d8f is 50, key is test_row_0/B:col10/1733500871837/Put/seqid=0 2024-12-06T16:01:12,467 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206fa763dd79aa14fd8b81333ff397525b1_0d7db0f41a15490076613839fa83ecf9 store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:12,472 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206fa763dd79aa14fd8b81333ff397525b1_0d7db0f41a15490076613839fa83ecf9, store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:12,472 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206fa763dd79aa14fd8b81333ff397525b1_0d7db0f41a15490076613839fa83ecf9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-06T16:01:12,474 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:01:12,475 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:01:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-12-06T16:01:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-06T16:01:12,480 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:01:12,481 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:01:12,482 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:01:12,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741987_1163 (size=12207) 2024-12-06T16:01:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:12,516 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:01:12,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d7db0f41a15490076613839fa83ecf9 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-06T16:01:12,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741988_1164 (size=4469) 2024-12-06T16:01:12,524 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d7db0f41a15490076613839fa83ecf9#A#compaction#141 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:12,525 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/c769a6a7841a46778d88e6691dd2c563 is 175, key is test_row_0/A:col10/1733500871837/Put/seqid=0 2024-12-06T16:01:12,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500932533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:12,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:12,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206d75c5957a69548dfbd5778787156bc26_0d7db0f41a15490076613839fa83ecf9 is 50, key is test_row_0/A:col10/1733500872516/Put/seqid=0 2024-12-06T16:01:12,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500932534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:12,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:12,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500932542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:12,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60246 deadline: 1733500932542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:12,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:12,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733500932542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:12,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:12,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:12,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:12,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741989_1165 (size=31161) 2024-12-06T16:01:12,561 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/c769a6a7841a46778d88e6691dd2c563 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/c769a6a7841a46778d88e6691dd2c563 2024-12-06T16:01:12,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741990_1166 (size=17034) 2024-12-06T16:01:12,569 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:12,569 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d7db0f41a15490076613839fa83ecf9/A of 0d7db0f41a15490076613839fa83ecf9 into c769a6a7841a46778d88e6691dd2c563(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:12,569 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:12,569 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., storeName=0d7db0f41a15490076613839fa83ecf9/A, priority=13, startTime=1733500872432; duration=0sec 2024-12-06T16:01:12,569 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:12,569 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d7db0f41a15490076613839fa83ecf9:A 2024-12-06T16:01:12,569 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:12,571 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:12,571 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 0d7db0f41a15490076613839fa83ecf9/C is initiating minor compaction (all files) 2024-12-06T16:01:12,571 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d7db0f41a15490076613839fa83ecf9/C in TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:12,571 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/52557655d8aa4afa82e74c90821660ae, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/57da6b55f1fb42e8a1c9cca47303615b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/94b9cfe17d574644b9ea90313d728233] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp, totalSize=35.3 K 2024-12-06T16:01:12,572 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52557655d8aa4afa82e74c90821660ae, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733500869576 2024-12-06T16:01:12,574 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57da6b55f1fb42e8a1c9cca47303615b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733500871487 2024-12-06T16:01:12,574 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94b9cfe17d574644b9ea90313d728233, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733500871507 2024-12-06T16:01:12,575 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206d75c5957a69548dfbd5778787156bc26_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206d75c5957a69548dfbd5778787156bc26_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-06T16:01:12,584 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/a0e5c9dceb734e628ae9f90909e9cf9b, store: [table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:12,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/a0e5c9dceb734e628ae9f90909e9cf9b is 175, key is test_row_0/A:col10/1733500872516/Put/seqid=0 2024-12-06T16:01:12,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741991_1167 (size=48139) 2024-12-06T16:01:12,605 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d7db0f41a15490076613839fa83ecf9#C#compaction#144 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:12,606 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=123, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/a0e5c9dceb734e628ae9f90909e9cf9b 2024-12-06T16:01:12,606 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/f4d8dde07e3448a09322ea1beffc2f0f is 50, key is test_row_0/C:col10/1733500871837/Put/seqid=0 2024-12-06T16:01:12,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741992_1168 (size=12207) 2024-12-06T16:01:12,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-06T16:01:12,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:12,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:12,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:12,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:12,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:12,641 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/f4d8dde07e3448a09322ea1beffc2f0f as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/f4d8dde07e3448a09322ea1beffc2f0f 2024-12-06T16:01:12,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500932643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:12,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/4c4524a231af4ccebcea8e03afbf071b is 50, key is test_row_0/B:col10/1733500872516/Put/seqid=0 2024-12-06T16:01:12,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:12,654 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d7db0f41a15490076613839fa83ecf9/C of 0d7db0f41a15490076613839fa83ecf9 into f4d8dde07e3448a09322ea1beffc2f0f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:12,654 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:12,654 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., storeName=0d7db0f41a15490076613839fa83ecf9/C, priority=13, startTime=1733500872436; duration=0sec 2024-12-06T16:01:12,655 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:12,655 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d7db0f41a15490076613839fa83ecf9:C 2024-12-06T16:01:12,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500932652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:12,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:12,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500932652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:12,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:12,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741993_1169 (size=12001) 2024-12-06T16:01:12,697 INFO [master/85bef17d9292:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-06T16:01:12,698 INFO [master/85bef17d9292:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-06T16:01:12,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-06T16:01:12,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:12,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:12,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:12,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:12,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:12,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-06T16:01:12,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:12,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:12,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500932851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:12,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:12,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:12,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500932862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:12,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:12,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500932862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:12,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:12,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:12,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:12,919 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/4067755f0a174bf7987fe9d23a8d2d8f as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/4067755f0a174bf7987fe9d23a8d2d8f 2024-12-06T16:01:12,926 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d7db0f41a15490076613839fa83ecf9/B of 0d7db0f41a15490076613839fa83ecf9 into 4067755f0a174bf7987fe9d23a8d2d8f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:12,926 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:12,926 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., storeName=0d7db0f41a15490076613839fa83ecf9/B, priority=13, startTime=1733500872433; duration=0sec 2024-12-06T16:01:12,926 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:12,927 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d7db0f41a15490076613839fa83ecf9:B 2024-12-06T16:01:12,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-06T16:01:12,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:12,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:12,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:12,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:12,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:12,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:13,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:13,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60246 deadline: 1733500933052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:13,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:13,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733500933052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:13,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/4c4524a231af4ccebcea8e03afbf071b 2024-12-06T16:01:13,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/986e48a09c694262ba7228d89f90b3fd is 50, key is test_row_0/C:col10/1733500872516/Put/seqid=0 2024-12-06T16:01:13,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741994_1170 (size=12001) 2024-12-06T16:01:13,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-06T16:01:13,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:13,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:13,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:13,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:13,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:13,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-06T16:01:13,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:13,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:13,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500933162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:13,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:13,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:13,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:13,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:13,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500933172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:13,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:13,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500933172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:13,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:13,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:13,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:13,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:13,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:13,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:13,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-06T16:01:13,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:13,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:13,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:13,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:13,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:13,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:13,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-06T16:01:13,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:13,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:13,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:13,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:13,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:13,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:13,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/986e48a09c694262ba7228d89f90b3fd 2024-12-06T16:01:13,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/a0e5c9dceb734e628ae9f90909e9cf9b as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/a0e5c9dceb734e628ae9f90909e9cf9b 2024-12-06T16:01:13,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/a0e5c9dceb734e628ae9f90909e9cf9b, entries=250, sequenceid=123, filesize=47.0 K 2024-12-06T16:01:13,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/4c4524a231af4ccebcea8e03afbf071b as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/4c4524a231af4ccebcea8e03afbf071b 2024-12-06T16:01:13,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/4c4524a231af4ccebcea8e03afbf071b, entries=150, sequenceid=123, filesize=11.7 K 2024-12-06T16:01:13,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/986e48a09c694262ba7228d89f90b3fd as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/986e48a09c694262ba7228d89f90b3fd 2024-12-06T16:01:13,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/986e48a09c694262ba7228d89f90b3fd, entries=150, sequenceid=123, filesize=11.7 K 2024-12-06T16:01:13,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 0d7db0f41a15490076613839fa83ecf9 in 998ms, sequenceid=123, compaction requested=false 2024-12-06T16:01:13,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:13,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-06T16:01:13,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:13,568 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 0d7db0f41a15490076613839fa83ecf9 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-12-06T16:01:13,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412069b53c6a980874f11a8bf680b97fe49bd_0d7db0f41a15490076613839fa83ecf9 is 50, key is test_row_0/A:col10/1733500872533/Put/seqid=0 2024-12-06T16:01:13,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741995_1171 (size=12304) 2024-12-06T16:01:13,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-06T16:01:13,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:13,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:13,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:13,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500933734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:13,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:13,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:13,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:13,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:13,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500933741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:13,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:13,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:13,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:13,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:13,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500933743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:13,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:13,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:13,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:13,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:13,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500933852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:13,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:13,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:13,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:13,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:13,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500933859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:13,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:13,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500933859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:13,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:13,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:13,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:14,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:14,005 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412069b53c6a980874f11a8bf680b97fe49bd_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412069b53c6a980874f11a8bf680b97fe49bd_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:14,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/d405fb48fcf14f6590bcb1600e752d87, store: [table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:14,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/d405fb48fcf14f6590bcb1600e752d87 is 175, key is test_row_0/A:col10/1733500872533/Put/seqid=0 2024-12-06T16:01:14,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741996_1172 (size=31105) 2024-12-06T16:01:14,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60246 deadline: 1733500934062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:14,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:14,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733500934063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:14,067 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 85bef17d9292,46105,1733500827732 for 'test_row_2' in 0d7db0f41a15490076613839fa83ecf9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T16:01:14,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:14,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:14,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:14,067 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 85bef17d9292,46105,1733500827732 for 'test_row_1' in 0d7db0f41a15490076613839fa83ecf9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T16:01:14,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:14,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:14,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:14,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:14,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500934063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:14,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:14,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:14,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:14,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:14,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500934072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:14,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:14,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:14,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500934072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:14,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:14,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:14,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:14,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:14,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500934381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:14,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:14,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:14,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:14,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:14,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500934382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:14,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:14,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500934382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:14,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:14,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:14,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:14,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:14,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:14,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:14,455 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/d405fb48fcf14f6590bcb1600e752d87 2024-12-06T16:01:14,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/78c0b399d64742d197f2919007511ef2 is 50, key is test_row_0/B:col10/1733500872533/Put/seqid=0 2024-12-06T16:01:14,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741997_1173 (size=12151) 2024-12-06T16:01:14,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-06T16:01:14,886 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/78c0b399d64742d197f2919007511ef2 2024-12-06T16:01:14,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/0844460a4d0d4f5fb639d68bd8125ec6 is 50, key is test_row_0/C:col10/1733500872533/Put/seqid=0 2024-12-06T16:01:14,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:14,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500934892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:14,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:14,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500934892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:14,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:14,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500934892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:14,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:14,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:14,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:14,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:14,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:14,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:14,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:14,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:14,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:14,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741998_1174 (size=12151) 2024-12-06T16:01:15,306 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/0844460a4d0d4f5fb639d68bd8125ec6 2024-12-06T16:01:15,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/d405fb48fcf14f6590bcb1600e752d87 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/d405fb48fcf14f6590bcb1600e752d87 2024-12-06T16:01:15,324 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/d405fb48fcf14f6590bcb1600e752d87, entries=150, sequenceid=136, filesize=30.4 K 2024-12-06T16:01:15,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/78c0b399d64742d197f2919007511ef2 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/78c0b399d64742d197f2919007511ef2 2024-12-06T16:01:15,333 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/78c0b399d64742d197f2919007511ef2, entries=150, sequenceid=136, filesize=11.9 K 2024-12-06T16:01:15,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/0844460a4d0d4f5fb639d68bd8125ec6 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/0844460a4d0d4f5fb639d68bd8125ec6 2024-12-06T16:01:15,339 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/0844460a4d0d4f5fb639d68bd8125ec6, entries=150, sequenceid=136, filesize=11.9 K 2024-12-06T16:01:15,341 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 0d7db0f41a15490076613839fa83ecf9 in 1772ms, sequenceid=136, compaction requested=true 2024-12-06T16:01:15,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:15,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:15,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-12-06T16:01:15,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-12-06T16:01:15,346 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-06T16:01:15,346 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8610 sec 2024-12-06T16:01:15,350 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 2.8720 sec 2024-12-06T16:01:15,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:15,906 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:01:15,906 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d7db0f41a15490076613839fa83ecf9 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-12-06T16:01:15,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206f2818c888dfa4db988ee5532571d21d4_0d7db0f41a15490076613839fa83ecf9 is 50, key is test_row_0/A:col10/1733500873741/Put/seqid=0 2024-12-06T16:01:15,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:15,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500935920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:15,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:15,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:15,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:15,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:15,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500935920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:15,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:15,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:15,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:15,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:15,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500935928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:15,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:15,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:15,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:15,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741999_1175 (size=14794) 2024-12-06T16:01:15,943 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:15,949 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206f2818c888dfa4db988ee5532571d21d4_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206f2818c888dfa4db988ee5532571d21d4_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:15,951 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/5fe7830232724c0583a5f49314f525cc, store: [table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:15,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/5fe7830232724c0583a5f49314f525cc is 175, key is test_row_0/A:col10/1733500873741/Put/seqid=0 2024-12-06T16:01:15,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742000_1176 (size=39749) 2024-12-06T16:01:16,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500936032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:16,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:16,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500936033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:16,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:16,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500936041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:16,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:16,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:16,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:16,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:16,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733500936082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:16,084 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 85bef17d9292,46105,1733500827732 for 'test_row_1' in 0d7db0f41a15490076613839fa83ecf9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T16:01:16,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:16,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:16,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:16,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:16,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60246 deadline: 1733500936091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:16,098 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 85bef17d9292,46105,1733500827732 for 'test_row_2' in 0d7db0f41a15490076613839fa83ecf9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T16:01:16,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:16,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:16,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:16,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:16,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500936242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:16,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:16,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:16,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:16,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500936242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:16,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:16,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:16,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:16,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:16,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:16,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500936252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:16,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:16,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:16,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:16,363 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=163, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/5fe7830232724c0583a5f49314f525cc 2024-12-06T16:01:16,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/98f2d8f4920e4d5987ce6f577885d916 is 50, key is test_row_0/B:col10/1733500873741/Put/seqid=0 2024-12-06T16:01:16,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742001_1177 (size=12151) 2024-12-06T16:01:16,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:16,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500936551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:16,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:16,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:16,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:16,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500936551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:16,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:16,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:16,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:16,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:16,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:16,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500936562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:16,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:16,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:16,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:16,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-06T16:01:16,652 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:01:16,652 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:01:16,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-12-06T16:01:16,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-06T16:01:16,655 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:01:16,656 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:01:16,656 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:01:16,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-06T16:01:16,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-06T16:01:16,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:16,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:16,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:16,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:16,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:16,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:16,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/98f2d8f4920e4d5987ce6f577885d916 2024-12-06T16:01:16,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/a462be074bf84d7cb3e92b7ef3b92cce is 50, key is test_row_0/C:col10/1733500873741/Put/seqid=0 2024-12-06T16:01:16,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742002_1178 (size=12151) 2024-12-06T16:01:16,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-06T16:01:16,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:16,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:16,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:16,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:16,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:16,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-06T16:01:17,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:17,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500937072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:17,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:17,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500937072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:17,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:17,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500937072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:17,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:17,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:17,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:17,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:17,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:17,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:17,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:17,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:17,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:17,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-06T16:01:17,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:17,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:17,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:17,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:17,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:17,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:17,275 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/a462be074bf84d7cb3e92b7ef3b92cce 2024-12-06T16:01:17,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-06T16:01:17,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:17,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:17,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:17,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:17,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:17,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/5fe7830232724c0583a5f49314f525cc as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/5fe7830232724c0583a5f49314f525cc 2024-12-06T16:01:17,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-06T16:01:17,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:17,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/5fe7830232724c0583a5f49314f525cc, entries=200, sequenceid=163, filesize=38.8 K 2024-12-06T16:01:17,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/98f2d8f4920e4d5987ce6f577885d916 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/98f2d8f4920e4d5987ce6f577885d916 2024-12-06T16:01:17,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/98f2d8f4920e4d5987ce6f577885d916, entries=150, sequenceid=163, filesize=11.9 K 2024-12-06T16:01:17,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/a462be074bf84d7cb3e92b7ef3b92cce as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/a462be074bf84d7cb3e92b7ef3b92cce 2024-12-06T16:01:17,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/a462be074bf84d7cb3e92b7ef3b92cce, entries=150, sequenceid=163, filesize=11.9 K 2024-12-06T16:01:17,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 0d7db0f41a15490076613839fa83ecf9 in 1399ms, sequenceid=163, compaction requested=true 2024-12-06T16:01:17,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:17,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d7db0f41a15490076613839fa83ecf9:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:01:17,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:17,306 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T16:01:17,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d7db0f41a15490076613839fa83ecf9:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:01:17,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:17,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d7db0f41a15490076613839fa83ecf9:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:01:17,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:17,306 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T16:01:17,307 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150154 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T16:01:17,308 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T16:01:17,308 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 0d7db0f41a15490076613839fa83ecf9/A is initiating minor compaction (all files) 2024-12-06T16:01:17,308 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 0d7db0f41a15490076613839fa83ecf9/B is initiating minor compaction (all files) 2024-12-06T16:01:17,308 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d7db0f41a15490076613839fa83ecf9/A in TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:17,308 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d7db0f41a15490076613839fa83ecf9/B in TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:17,308 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/c769a6a7841a46778d88e6691dd2c563, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/a0e5c9dceb734e628ae9f90909e9cf9b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/d405fb48fcf14f6590bcb1600e752d87, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/5fe7830232724c0583a5f49314f525cc] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp, totalSize=146.6 K 2024-12-06T16:01:17,308 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/4067755f0a174bf7987fe9d23a8d2d8f, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/4c4524a231af4ccebcea8e03afbf071b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/78c0b399d64742d197f2919007511ef2, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/98f2d8f4920e4d5987ce6f577885d916] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp, totalSize=47.4 K 2024-12-06T16:01:17,308 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:17,308 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. files: [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/c769a6a7841a46778d88e6691dd2c563, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/a0e5c9dceb734e628ae9f90909e9cf9b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/d405fb48fcf14f6590bcb1600e752d87, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/5fe7830232724c0583a5f49314f525cc] 2024-12-06T16:01:17,308 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 4067755f0a174bf7987fe9d23a8d2d8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733500871507 2024-12-06T16:01:17,309 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting c769a6a7841a46778d88e6691dd2c563, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733500871507 2024-12-06T16:01:17,309 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c4524a231af4ccebcea8e03afbf071b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733500871882 2024-12-06T16:01:17,309 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0e5c9dceb734e628ae9f90909e9cf9b, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733500871875 2024-12-06T16:01:17,309 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 78c0b399d64742d197f2919007511ef2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733500872529 2024-12-06T16:01:17,310 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting d405fb48fcf14f6590bcb1600e752d87, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733500872529 2024-12-06T16:01:17,310 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 98f2d8f4920e4d5987ce6f577885d916, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733500873737 2024-12-06T16:01:17,310 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fe7830232724c0583a5f49314f525cc, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733500873727 2024-12-06T16:01:17,324 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:17,329 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d7db0f41a15490076613839fa83ecf9#B#compaction#153 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:17,332 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/2a54925edb164bb1b68ce1f4b9cb482a is 50, key is test_row_0/B:col10/1733500873741/Put/seqid=0 2024-12-06T16:01:17,340 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412069260d32c96f84ae58218d123cfce09f8_0d7db0f41a15490076613839fa83ecf9 store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:17,344 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412069260d32c96f84ae58218d123cfce09f8_0d7db0f41a15490076613839fa83ecf9, store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:17,344 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412069260d32c96f84ae58218d123cfce09f8_0d7db0f41a15490076613839fa83ecf9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:17,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742003_1179 (size=12493) 2024-12-06T16:01:17,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742004_1180 (size=4469) 2024-12-06T16:01:17,380 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/2a54925edb164bb1b68ce1f4b9cb482a as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/2a54925edb164bb1b68ce1f4b9cb482a 2024-12-06T16:01:17,386 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0d7db0f41a15490076613839fa83ecf9/B of 0d7db0f41a15490076613839fa83ecf9 into 2a54925edb164bb1b68ce1f4b9cb482a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:17,386 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:17,386 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., storeName=0d7db0f41a15490076613839fa83ecf9/B, priority=12, startTime=1733500877306; duration=0sec 2024-12-06T16:01:17,388 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:17,389 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d7db0f41a15490076613839fa83ecf9:B 2024-12-06T16:01:17,389 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T16:01:17,392 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T16:01:17,392 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 0d7db0f41a15490076613839fa83ecf9/C is initiating minor compaction (all files) 2024-12-06T16:01:17,392 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d7db0f41a15490076613839fa83ecf9/C in TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:17,392 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/f4d8dde07e3448a09322ea1beffc2f0f, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/986e48a09c694262ba7228d89f90b3fd, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/0844460a4d0d4f5fb639d68bd8125ec6, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/a462be074bf84d7cb3e92b7ef3b92cce] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp, totalSize=47.4 K 2024-12-06T16:01:17,392 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting f4d8dde07e3448a09322ea1beffc2f0f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733500871507 2024-12-06T16:01:17,393 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 986e48a09c694262ba7228d89f90b3fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733500871882 2024-12-06T16:01:17,393 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 0844460a4d0d4f5fb639d68bd8125ec6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733500872529 2024-12-06T16:01:17,394 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting a462be074bf84d7cb3e92b7ef3b92cce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733500873737 2024-12-06T16:01:17,410 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d7db0f41a15490076613839fa83ecf9#C#compaction#155 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:17,410 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/5d70fe7d37af490e8bd513e3697c4dfb is 50, key is test_row_0/C:col10/1733500873741/Put/seqid=0 2024-12-06T16:01:17,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742005_1181 (size=12493) 2024-12-06T16:01:17,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-06T16:01:17,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:17,448 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 0d7db0f41a15490076613839fa83ecf9 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-12-06T16:01:17,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206a976dab4463a4211bd3560ebd46c269d_0d7db0f41a15490076613839fa83ecf9 is 50, key is test_row_0/A:col10/1733500875915/Put/seqid=0 2024-12-06T16:01:17,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742006_1182 (size=12304) 2024-12-06T16:01:17,772 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d7db0f41a15490076613839fa83ecf9#A#compaction#154 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:17,773 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/7e7c56ae6d46487490ad1b767bc9502f is 175, key is test_row_0/A:col10/1733500873741/Put/seqid=0 2024-12-06T16:01:17,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742007_1183 (size=31447) 2024-12-06T16:01:17,789 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/7e7c56ae6d46487490ad1b767bc9502f as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/7e7c56ae6d46487490ad1b767bc9502f 2024-12-06T16:01:17,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-06T16:01:17,797 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0d7db0f41a15490076613839fa83ecf9/A of 0d7db0f41a15490076613839fa83ecf9 into 7e7c56ae6d46487490ad1b767bc9502f(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:17,797 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:17,797 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., storeName=0d7db0f41a15490076613839fa83ecf9/A, priority=12, startTime=1733500877306; duration=0sec 2024-12-06T16:01:17,798 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:17,798 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d7db0f41a15490076613839fa83ecf9:A 2024-12-06T16:01:17,841 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/5d70fe7d37af490e8bd513e3697c4dfb as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/5d70fe7d37af490e8bd513e3697c4dfb 2024-12-06T16:01:17,849 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0d7db0f41a15490076613839fa83ecf9/C of 0d7db0f41a15490076613839fa83ecf9 into 5d70fe7d37af490e8bd513e3697c4dfb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:17,849 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:17,849 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., storeName=0d7db0f41a15490076613839fa83ecf9/C, priority=12, startTime=1733500877306; duration=0sec 2024-12-06T16:01:17,849 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:17,849 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d7db0f41a15490076613839fa83ecf9:C 2024-12-06T16:01:17,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:17,867 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206a976dab4463a4211bd3560ebd46c269d_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a976dab4463a4211bd3560ebd46c269d_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:17,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/50f91a0b5b7143a0b39b07bf16c55f51, store: [table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:17,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/50f91a0b5b7143a0b39b07bf16c55f51 is 175, key is test_row_0/A:col10/1733500875915/Put/seqid=0 2024-12-06T16:01:17,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742008_1184 (size=31105) 2024-12-06T16:01:17,882 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/50f91a0b5b7143a0b39b07bf16c55f51 2024-12-06T16:01:17,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/ab160feb28594e1ea85b48e535980f80 is 50, key is test_row_0/B:col10/1733500875915/Put/seqid=0 2024-12-06T16:01:17,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742009_1185 (size=12151) 2024-12-06T16:01:17,928 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/ab160feb28594e1ea85b48e535980f80 2024-12-06T16:01:17,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/c75ebfccc4ef4226addb1903ef797655 is 50, key is test_row_0/C:col10/1733500875915/Put/seqid=0 2024-12-06T16:01:17,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742010_1186 (size=12151) 2024-12-06T16:01:18,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:18,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:18,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:18,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500938134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:18,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:18,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:18,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:18,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:18,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500938137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:18,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:18,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500938139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:18,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500938252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:18,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:18,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:18,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:18,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500938252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:18,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:18,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:18,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:18,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500938252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:18,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:18,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:18,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:18,363 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/c75ebfccc4ef4226addb1903ef797655 2024-12-06T16:01:18,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/50f91a0b5b7143a0b39b07bf16c55f51 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/50f91a0b5b7143a0b39b07bf16c55f51 2024-12-06T16:01:18,376 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/50f91a0b5b7143a0b39b07bf16c55f51, entries=150, sequenceid=173, filesize=30.4 K 2024-12-06T16:01:18,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/ab160feb28594e1ea85b48e535980f80 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/ab160feb28594e1ea85b48e535980f80 2024-12-06T16:01:18,383 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/ab160feb28594e1ea85b48e535980f80, entries=150, sequenceid=173, filesize=11.9 K 2024-12-06T16:01:18,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/c75ebfccc4ef4226addb1903ef797655 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/c75ebfccc4ef4226addb1903ef797655 2024-12-06T16:01:18,391 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/c75ebfccc4ef4226addb1903ef797655, entries=150, sequenceid=173, filesize=11.9 K 2024-12-06T16:01:18,392 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 0d7db0f41a15490076613839fa83ecf9 in 944ms, sequenceid=173, compaction requested=false 2024-12-06T16:01:18,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:18,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:18,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-12-06T16:01:18,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-12-06T16:01:18,397 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-06T16:01:18,397 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7380 sec 2024-12-06T16:01:18,402 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.7460 sec 2024-12-06T16:01:18,469 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:01:18,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d7db0f41a15490076613839fa83ecf9 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-12-06T16:01:18,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:18,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:18,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:18,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500938474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:18,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500938480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:18,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:18,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500938483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:18,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:18,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:18,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:18,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120690224034a8b64b0ea46afbfa0643f0d7_0d7db0f41a15490076613839fa83ecf9 is 50, key is test_row_0/A:col10/1733500878137/Put/seqid=0 2024-12-06T16:01:18,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742011_1187 (size=14794) 2024-12-06T16:01:18,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500938592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:18,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:18,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500938592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:18,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:18,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:18,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:18,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500938602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:18,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:18,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-06T16:01:18,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:18,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500938804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:18,805 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:01:18,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:18,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:18,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:18,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:18,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500938804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:18,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:18,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:18,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:18,807 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:01:18,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-06T16:01:18,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-06T16:01:18,810 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:01:18,811 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:01:18,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:01:18,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:18,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500938812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:18,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:18,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:18,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-06T16:01:18,917 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:18,927 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120690224034a8b64b0ea46afbfa0643f0d7_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120690224034a8b64b0ea46afbfa0643f0d7_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:18,932 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/f4a8f96525e24ae7ace5fd93aac058a6, store: [table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:18,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/f4a8f96525e24ae7ace5fd93aac058a6 is 175, key is test_row_0/A:col10/1733500878137/Put/seqid=0 2024-12-06T16:01:18,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742012_1188 (size=39749) 2024-12-06T16:01:18,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-06T16:01:18,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:18,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:18,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:18,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:18,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:18,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:19,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:19,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500939111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:19,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:19,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:19,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:19,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500939111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:19,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-06T16:01:19,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-06T16:01:19,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:19,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:19,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:19,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:19,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:19,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500939121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:19,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:19,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:19,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:19,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-06T16:01:19,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:19,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:19,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:19,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:19,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:19,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:19,340 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/f4a8f96525e24ae7ace5fd93aac058a6 2024-12-06T16:01:19,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/79d7c1cdc0834f0b97ceddaf3018f536 is 50, key is test_row_0/B:col10/1733500878137/Put/seqid=0 2024-12-06T16:01:19,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742013_1189 (size=12151) 2024-12-06T16:01:19,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/79d7c1cdc0834f0b97ceddaf3018f536 2024-12-06T16:01:19,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/2a355c02c5db47afb3ca192bf4cb7e1f is 50, key is test_row_0/C:col10/1733500878137/Put/seqid=0 2024-12-06T16:01:19,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742014_1190 (size=12151) 2024-12-06T16:01:19,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/2a355c02c5db47afb3ca192bf4cb7e1f 2024-12-06T16:01:19,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/f4a8f96525e24ae7ace5fd93aac058a6 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/f4a8f96525e24ae7ace5fd93aac058a6 2024-12-06T16:01:19,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/f4a8f96525e24ae7ace5fd93aac058a6, entries=200, sequenceid=204, filesize=38.8 K 2024-12-06T16:01:19,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/79d7c1cdc0834f0b97ceddaf3018f536 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/79d7c1cdc0834f0b97ceddaf3018f536 2024-12-06T16:01:19,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-06T16:01:19,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:19,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:19,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:19,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:19,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:19,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-06T16:01:19,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:19,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/79d7c1cdc0834f0b97ceddaf3018f536, entries=150, sequenceid=204, filesize=11.9 K 2024-12-06T16:01:19,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/2a355c02c5db47afb3ca192bf4cb7e1f as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/2a355c02c5db47afb3ca192bf4cb7e1f 2024-12-06T16:01:19,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/2a355c02c5db47afb3ca192bf4cb7e1f, entries=150, sequenceid=204, filesize=11.9 K 2024-12-06T16:01:19,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 0d7db0f41a15490076613839fa83ecf9 in 979ms, sequenceid=204, compaction requested=true 2024-12-06T16:01:19,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:19,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d7db0f41a15490076613839fa83ecf9:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:01:19,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:19,449 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:19,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d7db0f41a15490076613839fa83ecf9:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:01:19,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:19,449 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:19,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d7db0f41a15490076613839fa83ecf9:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:01:19,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:19,450 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:19,451 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 0d7db0f41a15490076613839fa83ecf9/A is initiating minor compaction (all files) 2024-12-06T16:01:19,451 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d7db0f41a15490076613839fa83ecf9/A in TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:19,451 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/7e7c56ae6d46487490ad1b767bc9502f, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/50f91a0b5b7143a0b39b07bf16c55f51, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/f4a8f96525e24ae7ace5fd93aac058a6] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp, totalSize=99.9 K 2024-12-06T16:01:19,451 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:19,451 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. files: [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/7e7c56ae6d46487490ad1b767bc9502f, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/50f91a0b5b7143a0b39b07bf16c55f51, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/f4a8f96525e24ae7ace5fd93aac058a6] 2024-12-06T16:01:19,451 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:19,452 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 0d7db0f41a15490076613839fa83ecf9/B is initiating minor compaction (all files) 2024-12-06T16:01:19,452 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d7db0f41a15490076613839fa83ecf9/B in TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:19,452 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/2a54925edb164bb1b68ce1f4b9cb482a, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/ab160feb28594e1ea85b48e535980f80, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/79d7c1cdc0834f0b97ceddaf3018f536] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp, totalSize=35.9 K 2024-12-06T16:01:19,452 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e7c56ae6d46487490ad1b767bc9502f, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733500873737 2024-12-06T16:01:19,454 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a54925edb164bb1b68ce1f4b9cb482a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733500873737 2024-12-06T16:01:19,454 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50f91a0b5b7143a0b39b07bf16c55f51, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733500875913 2024-12-06T16:01:19,455 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting ab160feb28594e1ea85b48e535980f80, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733500875913 2024-12-06T16:01:19,455 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4a8f96525e24ae7ace5fd93aac058a6, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733500878135 2024-12-06T16:01:19,455 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 79d7c1cdc0834f0b97ceddaf3018f536, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733500878135 2024-12-06T16:01:19,467 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:19,474 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d7db0f41a15490076613839fa83ecf9#B#compaction#163 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:19,475 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/8eab2601ac134edb8b5b99c074671e99 is 50, key is test_row_0/B:col10/1733500878137/Put/seqid=0 2024-12-06T16:01:19,480 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206a975a696c09f4abcae6acc5c0539ce8b_0d7db0f41a15490076613839fa83ecf9 store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:19,483 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206a975a696c09f4abcae6acc5c0539ce8b_0d7db0f41a15490076613839fa83ecf9, store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:19,483 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206a975a696c09f4abcae6acc5c0539ce8b_0d7db0f41a15490076613839fa83ecf9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:19,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742015_1191 (size=12595) 2024-12-06T16:01:19,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742016_1192 (size=4469) 2024-12-06T16:01:19,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-06T16:01:19,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:19,586 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 0d7db0f41a15490076613839fa83ecf9 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-12-06T16:01:19,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206be2e266758054a15a669b1832c61aa00_0d7db0f41a15490076613839fa83ecf9 is 50, key is test_row_0/A:col10/1733500878472/Put/seqid=0 2024-12-06T16:01:19,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:19,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:19,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742017_1193 (size=12304) 2024-12-06T16:01:19,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:19,647 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206be2e266758054a15a669b1832c61aa00_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206be2e266758054a15a669b1832c61aa00_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:19,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/dad50953b79947c9abd53321e5b1ade7, store: [table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:19,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/dad50953b79947c9abd53321e5b1ade7 is 175, key is test_row_0/A:col10/1733500878472/Put/seqid=0 2024-12-06T16:01:19,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742018_1194 (size=31105) 2024-12-06T16:01:19,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:19,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500939691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:19,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:19,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:19,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:19,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:19,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500939692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:19,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:19,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:19,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:19,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:19,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500939695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:19,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:19,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:19,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:19,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:19,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500939802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:19,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:19,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500939802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:19,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:19,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500939802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:19,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:19,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:19,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:19,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:19,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:19,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:19,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:19,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:19,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:19,915 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/8eab2601ac134edb8b5b99c074671e99 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/8eab2601ac134edb8b5b99c074671e99 2024-12-06T16:01:19,922 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d7db0f41a15490076613839fa83ecf9#A#compaction#162 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:19,923 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/ba0a969c3af5432093f956d11d01664d is 175, key is test_row_0/A:col10/1733500878137/Put/seqid=0 2024-12-06T16:01:19,925 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d7db0f41a15490076613839fa83ecf9/B of 0d7db0f41a15490076613839fa83ecf9 into 8eab2601ac134edb8b5b99c074671e99(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:19,925 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:19,925 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., storeName=0d7db0f41a15490076613839fa83ecf9/B, priority=13, startTime=1733500879449; duration=0sec 2024-12-06T16:01:19,925 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:19,925 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d7db0f41a15490076613839fa83ecf9:B 2024-12-06T16:01:19,926 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:19,927 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:19,927 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 0d7db0f41a15490076613839fa83ecf9/C is initiating minor compaction (all files) 2024-12-06T16:01:19,927 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d7db0f41a15490076613839fa83ecf9/C in TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:19,927 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/5d70fe7d37af490e8bd513e3697c4dfb, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/c75ebfccc4ef4226addb1903ef797655, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/2a355c02c5db47afb3ca192bf4cb7e1f] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp, totalSize=35.9 K 2024-12-06T16:01:19,928 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d70fe7d37af490e8bd513e3697c4dfb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733500873737 2024-12-06T16:01:19,928 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting c75ebfccc4ef4226addb1903ef797655, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733500875913 2024-12-06T16:01:19,928 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a355c02c5db47afb3ca192bf4cb7e1f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733500878135 2024-12-06T16:01:19,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-06T16:01:19,943 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d7db0f41a15490076613839fa83ecf9#C#compaction#165 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:19,944 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/f0a1b6aa37a346c6a6d1e7ceffc01a63 is 50, key is test_row_0/C:col10/1733500878137/Put/seqid=0 2024-12-06T16:01:19,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742019_1195 (size=31549) 2024-12-06T16:01:19,964 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/ba0a969c3af5432093f956d11d01664d as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/ba0a969c3af5432093f956d11d01664d 2024-12-06T16:01:19,973 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d7db0f41a15490076613839fa83ecf9/A of 0d7db0f41a15490076613839fa83ecf9 into ba0a969c3af5432093f956d11d01664d(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:19,973 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:19,973 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., storeName=0d7db0f41a15490076613839fa83ecf9/A, priority=13, startTime=1733500879449; duration=0sec 2024-12-06T16:01:19,973 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:19,973 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d7db0f41a15490076613839fa83ecf9:A 2024-12-06T16:01:19,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742020_1196 (size=12595) 2024-12-06T16:01:20,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:20,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500940022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:20,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:20,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:20,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500940022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:20,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:20,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:20,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500940023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:20,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:20,056 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/dad50953b79947c9abd53321e5b1ade7 2024-12-06T16:01:20,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/a16bd9eebef94a5eafaa843161b2f3f4 is 50, key is test_row_0/B:col10/1733500878472/Put/seqid=0 2024-12-06T16:01:20,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742021_1197 (size=12151) 2024-12-06T16:01:20,090 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/a16bd9eebef94a5eafaa843161b2f3f4 2024-12-06T16:01:20,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/073363fb242549e1ab10b2e7b02cf9aa is 50, key is test_row_0/C:col10/1733500878472/Put/seqid=0 2024-12-06T16:01:20,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742022_1198 (size=12151) 2024-12-06T16:01:20,118 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/073363fb242549e1ab10b2e7b02cf9aa 2024-12-06T16:01:20,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:20,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60290 deadline: 1733500940111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:20,119 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 85bef17d9292,46105,1733500827732 for 'test_row_1' in 0d7db0f41a15490076613839fa83ecf9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T16:01:20,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:20,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:20,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60246 deadline: 1733500940123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:20,124 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 85bef17d9292,46105,1733500827732 for 'test_row_2' in 0d7db0f41a15490076613839fa83ecf9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T16:01:20,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:20,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/dad50953b79947c9abd53321e5b1ade7 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/dad50953b79947c9abd53321e5b1ade7 2024-12-06T16:01:20,141 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/dad50953b79947c9abd53321e5b1ade7, entries=150, sequenceid=211, filesize=30.4 K 2024-12-06T16:01:20,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/a16bd9eebef94a5eafaa843161b2f3f4 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/a16bd9eebef94a5eafaa843161b2f3f4 2024-12-06T16:01:20,151 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/a16bd9eebef94a5eafaa843161b2f3f4, entries=150, sequenceid=211, filesize=11.9 K 2024-12-06T16:01:20,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/073363fb242549e1ab10b2e7b02cf9aa as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/073363fb242549e1ab10b2e7b02cf9aa 2024-12-06T16:01:20,161 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/073363fb242549e1ab10b2e7b02cf9aa, entries=150, sequenceid=211, filesize=11.9 K 2024-12-06T16:01:20,165 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=174.43 KB/178620 for 0d7db0f41a15490076613839fa83ecf9 in 578ms, sequenceid=211, compaction requested=false 2024-12-06T16:01:20,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:20,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:20,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-06T16:01:20,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-06T16:01:20,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-06T16:01:20,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3560 sec 2024-12-06T16:01:20,171 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.3620 sec 2024-12-06T16:01:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:20,336 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:01:20,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d7db0f41a15490076613839fa83ecf9 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-12-06T16:01:20,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:20,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500940339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:20,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:20,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500940341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:20,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500940341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:20,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206dde3b68deda2443091ad6b021b783d9d_0d7db0f41a15490076613839fa83ecf9 is 50, key is test_row_0/A:col10/1733500880336/Put/seqid=0 2024-12-06T16:01:20,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742023_1199 (size=12304) 2024-12-06T16:01:20,383 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:20,392 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206dde3b68deda2443091ad6b021b783d9d_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206dde3b68deda2443091ad6b021b783d9d_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:20,394 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/f0a1b6aa37a346c6a6d1e7ceffc01a63 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/f0a1b6aa37a346c6a6d1e7ceffc01a63 2024-12-06T16:01:20,395 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/6099cde958d24df38024e5b59388e69b, store: [table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:20,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/6099cde958d24df38024e5b59388e69b is 175, key is test_row_0/A:col10/1733500880336/Put/seqid=0 2024-12-06T16:01:20,411 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d7db0f41a15490076613839fa83ecf9/C of 0d7db0f41a15490076613839fa83ecf9 into f0a1b6aa37a346c6a6d1e7ceffc01a63(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:20,411 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:20,411 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., storeName=0d7db0f41a15490076613839fa83ecf9/C, priority=13, startTime=1733500879449; duration=0sec 2024-12-06T16:01:20,412 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:20,412 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d7db0f41a15490076613839fa83ecf9:C 2024-12-06T16:01:20,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742024_1200 (size=31105) 2024-12-06T16:01:20,423 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=245, memsize=64.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/6099cde958d24df38024e5b59388e69b 2024-12-06T16:01:20,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/1961566d2993404c8d2a969963088721 is 50, key is test_row_0/B:col10/1733500880336/Put/seqid=0 2024-12-06T16:01:20,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742025_1201 (size=12151) 2024-12-06T16:01:20,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:20,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500940452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:20,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:20,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500940462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:20,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:20,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500940462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:20,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:20,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:20,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500940662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:20,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:20,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:20,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500940672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:20,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:20,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500940672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:20,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:20,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:20,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/1961566d2993404c8d2a969963088721 2024-12-06T16:01:20,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/777e05c1f7e24416ad60d2beccafad6c is 50, key is test_row_0/C:col10/1733500880336/Put/seqid=0 2024-12-06T16:01:20,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742026_1202 (size=12151) 2024-12-06T16:01:20,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-06T16:01:20,951 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:01:20,952 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:01:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-06T16:01:20,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-06T16:01:20,954 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:01:20,955 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:01:20,955 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:01:20,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:20,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500940972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:20,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:20,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:20,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500940982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:20,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:20,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500940982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:20,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:20,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:20,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:21,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-06T16:01:21,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-06T16:01:21,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:21,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:21,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:21,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:21,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:21,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:21,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-06T16:01:21,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:21,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:21,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:21,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:21,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:21,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:21,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-06T16:01:21,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/777e05c1f7e24416ad60d2beccafad6c 2024-12-06T16:01:21,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/6099cde958d24df38024e5b59388e69b as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/6099cde958d24df38024e5b59388e69b 2024-12-06T16:01:21,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/6099cde958d24df38024e5b59388e69b, entries=150, sequenceid=245, filesize=30.4 K 2024-12-06T16:01:21,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/1961566d2993404c8d2a969963088721 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/1961566d2993404c8d2a969963088721 2024-12-06T16:01:21,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/1961566d2993404c8d2a969963088721, entries=150, sequenceid=245, filesize=11.9 K 2024-12-06T16:01:21,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/777e05c1f7e24416ad60d2beccafad6c as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/777e05c1f7e24416ad60d2beccafad6c 2024-12-06T16:01:21,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/777e05c1f7e24416ad60d2beccafad6c, entries=150, sequenceid=245, filesize=11.9 K 2024-12-06T16:01:21,318 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~194.56 KB/199230, heapSize ~510.47 KB/522720, currentSize=13.42 KB/13740 for 0d7db0f41a15490076613839fa83ecf9 in 982ms, sequenceid=245, compaction requested=true 2024-12-06T16:01:21,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:21,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d7db0f41a15490076613839fa83ecf9:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:01:21,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:21,318 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:21,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d7db0f41a15490076613839fa83ecf9:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:01:21,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:21,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d7db0f41a15490076613839fa83ecf9:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:01:21,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-06T16:01:21,318 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:21,321 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:21,321 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93759 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:21,321 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 0d7db0f41a15490076613839fa83ecf9/B is initiating minor compaction (all files) 2024-12-06T16:01:21,321 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 0d7db0f41a15490076613839fa83ecf9/A is initiating minor compaction (all files) 2024-12-06T16:01:21,321 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d7db0f41a15490076613839fa83ecf9/A in TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:21,321 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d7db0f41a15490076613839fa83ecf9/B in TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:21,321 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/ba0a969c3af5432093f956d11d01664d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/dad50953b79947c9abd53321e5b1ade7, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/6099cde958d24df38024e5b59388e69b] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp, totalSize=91.6 K 2024-12-06T16:01:21,321 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/8eab2601ac134edb8b5b99c074671e99, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/a16bd9eebef94a5eafaa843161b2f3f4, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/1961566d2993404c8d2a969963088721] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp, totalSize=36.0 K 2024-12-06T16:01:21,321 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:21,321 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. files: [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/ba0a969c3af5432093f956d11d01664d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/dad50953b79947c9abd53321e5b1ade7, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/6099cde958d24df38024e5b59388e69b] 2024-12-06T16:01:21,322 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 8eab2601ac134edb8b5b99c074671e99, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733500878135 2024-12-06T16:01:21,322 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba0a969c3af5432093f956d11d01664d, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733500878135 2024-12-06T16:01:21,322 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting dad50953b79947c9abd53321e5b1ade7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733500878472 2024-12-06T16:01:21,323 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting a16bd9eebef94a5eafaa843161b2f3f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733500878472 2024-12-06T16:01:21,323 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6099cde958d24df38024e5b59388e69b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733500879681 2024-12-06T16:01:21,324 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 1961566d2993404c8d2a969963088721, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733500879681 2024-12-06T16:01:21,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,353 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d7db0f41a15490076613839fa83ecf9#B#compaction#171 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:21,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,353 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/fb2e3329863f43589d2a12f5e5139905 is 50, key is test_row_0/B:col10/1733500880336/Put/seqid=0 2024-12-06T16:01:21,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,360 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:21,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,381 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206ed1e54a88457496f94ae46d4e6ad7e3e_0d7db0f41a15490076613839fa83ecf9 store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:21,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,386 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206ed1e54a88457496f94ae46d4e6ad7e3e_0d7db0f41a15490076613839fa83ecf9, store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:21,386 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206ed1e54a88457496f94ae46d4e6ad7e3e_0d7db0f41a15490076613839fa83ecf9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:21,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742027_1203 (size=12697) 2024-12-06T16:01:21,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,417 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/fb2e3329863f43589d2a12f5e5139905 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/fb2e3329863f43589d2a12f5e5139905 2024-12-06T16:01:21,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-06T16:01:21,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:21,419 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 0d7db0f41a15490076613839fa83ecf9 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-12-06T16:01:21,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,424 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d7db0f41a15490076613839fa83ecf9/B of 0d7db0f41a15490076613839fa83ecf9 into fb2e3329863f43589d2a12f5e5139905(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:21,424 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:21,424 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., storeName=0d7db0f41a15490076613839fa83ecf9/B, priority=13, startTime=1733500881318; duration=0sec 2024-12-06T16:01:21,424 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:21,424 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d7db0f41a15490076613839fa83ecf9:B 2024-12-06T16:01:21,424 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:21,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,426 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:21,426 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 0d7db0f41a15490076613839fa83ecf9/C is initiating minor compaction (all files) 2024-12-06T16:01:21,426 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d7db0f41a15490076613839fa83ecf9/C in TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:21,426 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/f0a1b6aa37a346c6a6d1e7ceffc01a63, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/073363fb242549e1ab10b2e7b02cf9aa, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/777e05c1f7e24416ad60d2beccafad6c] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp, totalSize=36.0 K 2024-12-06T16:01:21,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,426 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting f0a1b6aa37a346c6a6d1e7ceffc01a63, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733500878135 2024-12-06T16:01:21,427 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 073363fb242549e1ab10b2e7b02cf9aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733500878472 2024-12-06T16:01:21,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,427 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 777e05c1f7e24416ad60d2beccafad6c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733500879681 2024-12-06T16:01:21,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742028_1204 (size=4469) 2024-12-06T16:01:21,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,473 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d7db0f41a15490076613839fa83ecf9#A#compaction#172 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:21,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,474 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/b59783c5d1f643eda31253faaa7b57b8 is 175, key is test_row_0/A:col10/1733500880336/Put/seqid=0 2024-12-06T16:01:21,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,490 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d7db0f41a15490076613839fa83ecf9#C#compaction#173 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:21,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,491 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/42d19f131cb34425a222fb715cb2f01b is 50, key is test_row_0/C:col10/1733500880336/Put/seqid=0 2024-12-06T16:01:21,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206806144f2f7424cf49e9fce14c9169510_0d7db0f41a15490076613839fa83ecf9 is 50, key is test_row_0/A:col10/1733500880339/Put/seqid=0 2024-12-06T16:01:21,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:21,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:21,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:21,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742029_1205 (size=31651) 2024-12-06T16:01:21,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742030_1206 (size=12697) 2024-12-06T16:01:21,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-06T16:01:21,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:21,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500941600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:21,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:21,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500941604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:21,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:21,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:21,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:21,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:21,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500941612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:21,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:21,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:21,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:21,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742031_1207 (size=9814) 2024-12-06T16:01:21,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:21,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500941711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:21,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:21,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:21,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:21,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:21,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500941722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:21,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:21,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500941723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:21,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:21,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:21,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:21,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:21,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:21,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:21,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500941921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:21,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:21,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:21,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:21,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500941932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:21,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500941932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:21,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:21,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:21,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:21,970 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/b59783c5d1f643eda31253faaa7b57b8 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/b59783c5d1f643eda31253faaa7b57b8 2024-12-06T16:01:21,979 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d7db0f41a15490076613839fa83ecf9/A of 0d7db0f41a15490076613839fa83ecf9 into b59783c5d1f643eda31253faaa7b57b8(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:21,979 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:21,979 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., storeName=0d7db0f41a15490076613839fa83ecf9/A, priority=13, startTime=1733500881318; duration=0sec 2024-12-06T16:01:21,980 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:21,980 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d7db0f41a15490076613839fa83ecf9:A 2024-12-06T16:01:22,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,023 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/42d19f131cb34425a222fb715cb2f01b as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/42d19f131cb34425a222fb715cb2f01b 2024-12-06T16:01:22,025 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206806144f2f7424cf49e9fce14c9169510_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206806144f2f7424cf49e9fce14c9169510_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:22,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/1ed9be18977c4b73963df2240761adbc, store: [table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:22,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/1ed9be18977c4b73963df2240761adbc is 175, key is test_row_0/A:col10/1733500880339/Put/seqid=0 2024-12-06T16:01:22,031 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d7db0f41a15490076613839fa83ecf9/C of 0d7db0f41a15490076613839fa83ecf9 into 42d19f131cb34425a222fb715cb2f01b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:22,031 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:22,032 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., storeName=0d7db0f41a15490076613839fa83ecf9/C, priority=13, startTime=1733500881318; duration=0sec 2024-12-06T16:01:22,032 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:22,032 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d7db0f41a15490076613839fa83ecf9:C 2024-12-06T16:01:22,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742032_1208 (size=22461) 2024-12-06T16:01:22,037 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=251, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/1ed9be18977c4b73963df2240761adbc 2024-12-06T16:01:22,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/29a5cb3989544a04a6ef6329d1f8d551 is 50, key is test_row_0/B:col10/1733500880339/Put/seqid=0 2024-12-06T16:01:22,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742033_1209 (size=9757) 2024-12-06T16:01:22,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-06T16:01:22,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500942231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:22,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:22,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:22,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:22,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:22,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:22,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500942242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:22,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500942242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:22,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:22,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:22,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:22,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:22,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:22,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:22,495 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/29a5cb3989544a04a6ef6329d1f8d551 2024-12-06T16:01:22,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/6c28b13afd564c39a89f13d5567ad7c1 is 50, key is test_row_0/C:col10/1733500880339/Put/seqid=0 2024-12-06T16:01:22,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742034_1210 (size=9757) 2024-12-06T16:01:22,519 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/6c28b13afd564c39a89f13d5567ad7c1 2024-12-06T16:01:22,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/1ed9be18977c4b73963df2240761adbc as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/1ed9be18977c4b73963df2240761adbc 2024-12-06T16:01:22,536 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/1ed9be18977c4b73963df2240761adbc, entries=100, sequenceid=251, filesize=21.9 K 2024-12-06T16:01:22,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/29a5cb3989544a04a6ef6329d1f8d551 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/29a5cb3989544a04a6ef6329d1f8d551 2024-12-06T16:01:22,544 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/29a5cb3989544a04a6ef6329d1f8d551, entries=100, sequenceid=251, filesize=9.5 K 2024-12-06T16:01:22,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/6c28b13afd564c39a89f13d5567ad7c1 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/6c28b13afd564c39a89f13d5567ad7c1 2024-12-06T16:01:22,549 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/6c28b13afd564c39a89f13d5567ad7c1, entries=100, sequenceid=251, filesize=9.5 K 2024-12-06T16:01:22,551 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=187.85 KB/192360 for 0d7db0f41a15490076613839fa83ecf9 in 1131ms, sequenceid=251, compaction requested=false 2024-12-06T16:01:22,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:22,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:22,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-06T16:01:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-06T16:01:22,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-06T16:01:22,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5980 sec 2024-12-06T16:01:22,559 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.6050 sec 2024-12-06T16:01:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:22,744 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:01:22,744 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d7db0f41a15490076613839fa83ecf9 3/3 column families, dataSize=194.56 KB heapSize=510.52 KB 2024-12-06T16:01:22,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500942755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:22,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:22,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:22,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:22,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500942756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:22,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:22,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:22,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:22,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412063685169224cc40478718ccbc915f2a2a_0d7db0f41a15490076613839fa83ecf9 is 50, key is test_row_0/A:col10/1733500881598/Put/seqid=0 2024-12-06T16:01:22,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500942761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:22,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:22,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:22,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:22,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742035_1211 (size=14994) 2024-12-06T16:01:22,779 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,785 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412063685169224cc40478718ccbc915f2a2a_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412063685169224cc40478718ccbc915f2a2a_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:22,786 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/462f1aaa86bd402f8c8c830146768f77, store: [table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:22,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/462f1aaa86bd402f8c8c830146768f77 is 175, key is test_row_0/A:col10/1733500881598/Put/seqid=0 2024-12-06T16:01:22,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742036_1212 (size=39949) 2024-12-06T16:01:22,820 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=286, memsize=64.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/462f1aaa86bd402f8c8c830146768f77 2024-12-06T16:01:22,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/ead077119c67435492b4adff00cb3bb9 is 50, key is test_row_0/B:col10/1733500881598/Put/seqid=0 2024-12-06T16:01:22,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742037_1213 (size=12301) 2024-12-06T16:01:22,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500942861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:22,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/ead077119c67435492b4adff00cb3bb9 2024-12-06T16:01:22,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:22,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:22,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:22,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500942863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:22,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:22,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:22,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:22,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/43434bb12fb447df883391ebeef96c59 is 50, key is test_row_0/C:col10/1733500881598/Put/seqid=0 2024-12-06T16:01:22,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742038_1214 (size=12301) 2024-12-06T16:01:22,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/43434bb12fb447df883391ebeef96c59 2024-12-06T16:01:22,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/462f1aaa86bd402f8c8c830146768f77 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/462f1aaa86bd402f8c8c830146768f77 2024-12-06T16:01:22,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/462f1aaa86bd402f8c8c830146768f77, entries=200, sequenceid=286, filesize=39.0 K 2024-12-06T16:01:22,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/ead077119c67435492b4adff00cb3bb9 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/ead077119c67435492b4adff00cb3bb9 2024-12-06T16:01:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/ead077119c67435492b4adff00cb3bb9, entries=150, sequenceid=286, filesize=12.0 K 2024-12-06T16:01:22,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/43434bb12fb447df883391ebeef96c59 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/43434bb12fb447df883391ebeef96c59 2024-12-06T16:01:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/43434bb12fb447df883391ebeef96c59, entries=150, sequenceid=286, filesize=12.0 K 2024-12-06T16:01:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,938 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~194.56 KB/199230, heapSize ~510.47 KB/522720, currentSize=13.42 KB/13740 for 0d7db0f41a15490076613839fa83ecf9 in 194ms, sequenceid=286, compaction requested=true 2024-12-06T16:01:22,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,939 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:22,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d7db0f41a15490076613839fa83ecf9:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:01:22,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,939 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,940 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:22,940 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 0d7db0f41a15490076613839fa83ecf9/B is initiating minor compaction (all files) 2024-12-06T16:01:22,940 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d7db0f41a15490076613839fa83ecf9/B in TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,941 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/fb2e3329863f43589d2a12f5e5139905, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/29a5cb3989544a04a6ef6329d1f8d551, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/ead077119c67435492b4adff00cb3bb9] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp, totalSize=33.9 K 2024-12-06T16:01:22,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d7db0f41a15490076613839fa83ecf9:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:01:22,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:22,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d7db0f41a15490076613839fa83ecf9:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:01:22,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,943 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting fb2e3329863f43589d2a12f5e5139905, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733500879681 2024-12-06T16:01:22,943 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,943 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 0d7db0f41a15490076613839fa83ecf9/A is initiating minor compaction (all files) 2024-12-06T16:01:22,943 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d7db0f41a15490076613839fa83ecf9/A in TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:22,943 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/b59783c5d1f643eda31253faaa7b57b8, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/1ed9be18977c4b73963df2240761adbc, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/462f1aaa86bd402f8c8c830146768f77] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp, totalSize=91.9 K 2024-12-06T16:01:22,943 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:22,943 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. files: [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/b59783c5d1f643eda31253faaa7b57b8, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/1ed9be18977c4b73963df2240761adbc, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/462f1aaa86bd402f8c8c830146768f77] 2024-12-06T16:01:22,943 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 29a5cb3989544a04a6ef6329d1f8d551, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733500880339 2024-12-06T16:01:22,944 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting b59783c5d1f643eda31253faaa7b57b8, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733500879681 2024-12-06T16:01:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,944 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting ead077119c67435492b4adff00cb3bb9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733500881598 2024-12-06T16:01:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,944 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ed9be18977c4b73963df2240761adbc, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733500880339 2024-12-06T16:01:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,945 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 462f1aaa86bd402f8c8c830146768f77, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733500881598 2024-12-06T16:01:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,958 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d7db0f41a15490076613839fa83ecf9#B#compaction#180 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,958 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/9e4d971794e74f05bedcb4a33904182c is 50, key is test_row_0/B:col10/1733500881598/Put/seqid=0 2024-12-06T16:01:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,960 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,972 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206e61df8819a8c409f9681e13286547d5d_0d7db0f41a15490076613839fa83ecf9 store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,975 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206e61df8819a8c409f9681e13286547d5d_0d7db0f41a15490076613839fa83ecf9, store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,976 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206e61df8819a8c409f9681e13286547d5d_0d7db0f41a15490076613839fa83ecf9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742039_1215 (size=12949) 2024-12-06T16:01:23,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742040_1216 (size=4469) 2024-12-06T16:01:23,009 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d7db0f41a15490076613839fa83ecf9#A#compaction#181 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:23,010 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/3d7cf3b8e68249449f5ebd0d75b6f59e is 175, key is test_row_0/A:col10/1733500881598/Put/seqid=0 2024-12-06T16:01:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742041_1217 (size=31903) 2024-12-06T16:01:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,038 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/9e4d971794e74f05bedcb4a33904182c as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/9e4d971794e74f05bedcb4a33904182c 2024-12-06T16:01:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,047 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/3d7cf3b8e68249449f5ebd0d75b6f59e as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/3d7cf3b8e68249449f5ebd0d75b6f59e 2024-12-06T16:01:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,049 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d7db0f41a15490076613839fa83ecf9/B of 0d7db0f41a15490076613839fa83ecf9 into 9e4d971794e74f05bedcb4a33904182c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:23,049 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:23,049 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., storeName=0d7db0f41a15490076613839fa83ecf9/B, priority=13, startTime=1733500882939; duration=0sec 2024-12-06T16:01:23,049 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:23,049 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d7db0f41a15490076613839fa83ecf9:B 2024-12-06T16:01:23,049 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,051 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:23,051 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 0d7db0f41a15490076613839fa83ecf9/C is initiating minor compaction (all files) 2024-12-06T16:01:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,051 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d7db0f41a15490076613839fa83ecf9/C in TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:23,051 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/42d19f131cb34425a222fb715cb2f01b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/6c28b13afd564c39a89f13d5567ad7c1, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/43434bb12fb447df883391ebeef96c59] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp, totalSize=33.9 K 2024-12-06T16:01:23,051 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 42d19f131cb34425a222fb715cb2f01b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733500879681 2024-12-06T16:01:23,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,052 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c28b13afd564c39a89f13d5567ad7c1, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733500880339 2024-12-06T16:01:23,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,053 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 43434bb12fb447df883391ebeef96c59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733500881598 2024-12-06T16:01:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,061 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d7db0f41a15490076613839fa83ecf9/A of 0d7db0f41a15490076613839fa83ecf9 into 3d7cf3b8e68249449f5ebd0d75b6f59e(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:23,061 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:23,061 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., storeName=0d7db0f41a15490076613839fa83ecf9/A, priority=13, startTime=1733500882939; duration=0sec 2024-12-06T16:01:23,061 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:23,061 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d7db0f41a15490076613839fa83ecf9:A 2024-12-06T16:01:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,086 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d7db0f41a15490076613839fa83ecf9#C#compaction#182 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,087 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/6b070626adf84ee4b4d5ad595bc1ef96 is 50, key is test_row_0/C:col10/1733500881598/Put/seqid=0 2024-12-06T16:01:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-06T16:01:23,113 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:01:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,115 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:01:23,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-06T16:01:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,118 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:01:23,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,118 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:01:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,119 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:01:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-06T16:01:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742042_1218 (size=12949) 2024-12-06T16:01:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,139 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/6b070626adf84ee4b4d5ad595bc1ef96 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/6b070626adf84ee4b4d5ad595bc1ef96 2024-12-06T16:01:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,146 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:01:23,146 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d7db0f41a15490076613839fa83ecf9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-06T16:01:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:23,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,163 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d7db0f41a15490076613839fa83ecf9/C of 0d7db0f41a15490076613839fa83ecf9 into 6b070626adf84ee4b4d5ad595bc1ef96(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:23,163 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:23,163 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., storeName=0d7db0f41a15490076613839fa83ecf9/C, priority=13, startTime=1733500882941; duration=0sec 2024-12-06T16:01:23,164 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:23,164 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d7db0f41a15490076613839fa83ecf9:C 2024-12-06T16:01:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412064d6dd87f43b24a2cb7499bccdc772161_0d7db0f41a15490076613839fa83ecf9 is 50, key is test_row_0/A:col10/1733500883137/Put/seqid=0 2024-12-06T16:01:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-06T16:01:23,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742043_1219 (size=22618) 2024-12-06T16:01:23,228 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:23,234 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412064d6dd87f43b24a2cb7499bccdc772161_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412064d6dd87f43b24a2cb7499bccdc772161_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:23,236 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/e42c7f0fd4494750a8cc6b0ef3cb276b, store: [table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:23,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/e42c7f0fd4494750a8cc6b0ef3cb276b is 175, key is test_row_0/A:col10/1733500883137/Put/seqid=0 2024-12-06T16:01:23,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742044_1220 (size=66023) 2024-12-06T16:01:23,263 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=299, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/e42c7f0fd4494750a8cc6b0ef3cb276b 2024-12-06T16:01:23,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-06T16:01:23,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:23,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:23,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:23,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:23,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:23,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/11fcccfb86e04c1ebfd4e98349aaa86d is 50, key is test_row_0/B:col10/1733500883137/Put/seqid=0 2024-12-06T16:01:23,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500943277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:23,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:23,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500943279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:23,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:23,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:23,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:23,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:23,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:23,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742045_1221 (size=12301) 2024-12-06T16:01:23,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:23,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500943391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:23,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:23,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500943391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:23,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:23,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:23,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:23,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-06T16:01:23,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:23,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:23,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:23,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:23,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:23,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:23,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-06T16:01:23,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-06T16:01:23,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:23,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:23,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:23,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:23,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:23,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:23,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:23,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500943611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:23,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:23,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:23,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:23,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:23,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500943612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:23,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:23,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:23,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:23,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-06T16:01:23,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:23,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:23,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:23,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/11fcccfb86e04c1ebfd4e98349aaa86d 2024-12-06T16:01:23,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:23,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:23,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-06T16:01:23,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/456dd17bb2d842c2b4a0c0af1f4b5003 is 50, key is test_row_0/C:col10/1733500883137/Put/seqid=0 2024-12-06T16:01:23,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742046_1222 (size=12301) 2024-12-06T16:01:23,762 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/456dd17bb2d842c2b4a0c0af1f4b5003 2024-12-06T16:01:23,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/e42c7f0fd4494750a8cc6b0ef3cb276b as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/e42c7f0fd4494750a8cc6b0ef3cb276b 2024-12-06T16:01:23,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/e42c7f0fd4494750a8cc6b0ef3cb276b, entries=350, sequenceid=299, filesize=64.5 K 2024-12-06T16:01:23,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/11fcccfb86e04c1ebfd4e98349aaa86d as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/11fcccfb86e04c1ebfd4e98349aaa86d 2024-12-06T16:01:23,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:23,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500943782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:23,785 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 85bef17d9292,46105,1733500827732 for 'test_row_2' in 0d7db0f41a15490076613839fa83ecf9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T16:01:23,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:23,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:23,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:23,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/11fcccfb86e04c1ebfd4e98349aaa86d, entries=150, sequenceid=299, filesize=12.0 K 2024-12-06T16:01:23,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/456dd17bb2d842c2b4a0c0af1f4b5003 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/456dd17bb2d842c2b4a0c0af1f4b5003 2024-12-06T16:01:23,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/456dd17bb2d842c2b4a0c0af1f4b5003, entries=150, sequenceid=299, filesize=12.0 K 2024-12-06T16:01:23,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 0d7db0f41a15490076613839fa83ecf9 in 659ms, sequenceid=299, compaction requested=false 2024-12-06T16:01:23,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:23,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-06T16:01:23,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:23,901 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 0d7db0f41a15490076613839fa83ecf9 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-06T16:01:23,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206b2bcd1c2a68b4e5ea20c4fe9d4bbdea8_0d7db0f41a15490076613839fa83ecf9 is 50, key is test_row_0/A:col10/1733500883263/Put/seqid=0 2024-12-06T16:01:23,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742047_1223 (size=12454) 2024-12-06T16:01:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:23,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:23,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500943940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:23,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:23,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:23,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:23,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500943941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:23,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:23,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:23,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:24,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500944051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:24,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:24,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500944051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:24,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:24,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:24,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:24,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-06T16:01:24,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:24,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500944261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:24,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:24,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500944261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:24,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:24,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:24,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:24,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:24,327 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206b2bcd1c2a68b4e5ea20c4fe9d4bbdea8_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206b2bcd1c2a68b4e5ea20c4fe9d4bbdea8_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:24,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/a2d4a2794538445a9c5fb82dd0069173, store: [table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:24,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/a2d4a2794538445a9c5fb82dd0069173 is 175, key is test_row_0/A:col10/1733500883263/Put/seqid=0 2024-12-06T16:01:24,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742048_1224 (size=31255) 2024-12-06T16:01:24,342 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=326, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/a2d4a2794538445a9c5fb82dd0069173 2024-12-06T16:01:24,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/272772f1e8904cbba890871cc952cd08 is 50, key is test_row_0/B:col10/1733500883263/Put/seqid=0 2024-12-06T16:01:24,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742049_1225 (size=12301) 2024-12-06T16:01:24,371 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/272772f1e8904cbba890871cc952cd08 2024-12-06T16:01:24,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/05283e34e4c1425f8299e7a4732245a9 is 50, key is test_row_0/C:col10/1733500883263/Put/seqid=0 2024-12-06T16:01:24,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742050_1226 (size=12301) 2024-12-06T16:01:24,400 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/05283e34e4c1425f8299e7a4732245a9 2024-12-06T16:01:24,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/a2d4a2794538445a9c5fb82dd0069173 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/a2d4a2794538445a9c5fb82dd0069173 2024-12-06T16:01:24,413 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/a2d4a2794538445a9c5fb82dd0069173, entries=150, sequenceid=326, filesize=30.5 K 2024-12-06T16:01:24,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/272772f1e8904cbba890871cc952cd08 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/272772f1e8904cbba890871cc952cd08 2024-12-06T16:01:24,422 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/272772f1e8904cbba890871cc952cd08, entries=150, sequenceid=326, filesize=12.0 K 2024-12-06T16:01:24,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/05283e34e4c1425f8299e7a4732245a9 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/05283e34e4c1425f8299e7a4732245a9 2024-12-06T16:01:24,431 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/05283e34e4c1425f8299e7a4732245a9, entries=150, sequenceid=326, filesize=12.0 K 2024-12-06T16:01:24,432 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 0d7db0f41a15490076613839fa83ecf9 in 532ms, sequenceid=326, compaction requested=true 2024-12-06T16:01:24,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:24,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:24,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-06T16:01:24,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-06T16:01:24,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-06T16:01:24,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3150 sec 2024-12-06T16:01:24,439 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.3230 sec 2024-12-06T16:01:24,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:24,574 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:01:24,574 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d7db0f41a15490076613839fa83ecf9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-06T16:01:24,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206509e3d48889a4ac4b88424131d3bcf8d_0d7db0f41a15490076613839fa83ecf9 is 50, key is test_row_0/A:col10/1733500883937/Put/seqid=0 2024-12-06T16:01:24,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742051_1227 (size=17534) 2024-12-06T16:01:24,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:24,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500944640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:24,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:24,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:24,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:24,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:24,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500944641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:24,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:24,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:24,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:24,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500944752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:24,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:24,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:24,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:24,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500944753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:24,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:24,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:24,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:24,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500944962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:24,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500944962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:24,994 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:25,001 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206509e3d48889a4ac4b88424131d3bcf8d_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206509e3d48889a4ac4b88424131d3bcf8d_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:25,002 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/d30d64d4a2fc462aa9c0cd734da62b85, store: [table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:25,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/d30d64d4a2fc462aa9c0cd734da62b85 is 175, key is test_row_0/A:col10/1733500883937/Put/seqid=0 2024-12-06T16:01:25,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742052_1228 (size=48639) 2024-12-06T16:01:25,022 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=338, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/d30d64d4a2fc462aa9c0cd734da62b85 2024-12-06T16:01:25,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/5f71aa6bfb00422a9c855ff5169ce48c is 50, key is test_row_0/B:col10/1733500883937/Put/seqid=0 2024-12-06T16:01:25,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742053_1229 (size=12301) 2024-12-06T16:01:25,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-06T16:01:25,264 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:01:25,264 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:01:25,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-06T16:01:25,267 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:01:25,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-06T16:01:25,270 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:01:25,271 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:01:25,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:25,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500945272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:25,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:25,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:25,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500945272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:25,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:25,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:25,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:25,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:25,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:25,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-06T16:01:25,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-06T16:01:25,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:25,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:25,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:25,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:25,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:25,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:25,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/5f71aa6bfb00422a9c855ff5169ce48c 2024-12-06T16:01:25,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/580cd27d4ee54406af0f618f398665ca is 50, key is test_row_0/C:col10/1733500883937/Put/seqid=0 2024-12-06T16:01:25,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742054_1230 (size=12301) 2024-12-06T16:01:25,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/580cd27d4ee54406af0f618f398665ca 2024-12-06T16:01:25,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/d30d64d4a2fc462aa9c0cd734da62b85 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/d30d64d4a2fc462aa9c0cd734da62b85 2024-12-06T16:01:25,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/d30d64d4a2fc462aa9c0cd734da62b85, entries=250, sequenceid=338, filesize=47.5 K 2024-12-06T16:01:25,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/5f71aa6bfb00422a9c855ff5169ce48c as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/5f71aa6bfb00422a9c855ff5169ce48c 2024-12-06T16:01:25,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/5f71aa6bfb00422a9c855ff5169ce48c, entries=150, sequenceid=338, filesize=12.0 K 2024-12-06T16:01:25,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/580cd27d4ee54406af0f618f398665ca as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/580cd27d4ee54406af0f618f398665ca 2024-12-06T16:01:25,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/580cd27d4ee54406af0f618f398665ca, entries=150, sequenceid=338, filesize=12.0 K 2024-12-06T16:01:25,508 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 0d7db0f41a15490076613839fa83ecf9 in 934ms, sequenceid=338, compaction requested=true 2024-12-06T16:01:25,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:25,508 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T16:01:25,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d7db0f41a15490076613839fa83ecf9:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:01:25,510 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 177820 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T16:01:25,512 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 0d7db0f41a15490076613839fa83ecf9/A is initiating minor compaction (all files) 2024-12-06T16:01:25,512 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d7db0f41a15490076613839fa83ecf9/A in TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:25,512 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/3d7cf3b8e68249449f5ebd0d75b6f59e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/e42c7f0fd4494750a8cc6b0ef3cb276b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/a2d4a2794538445a9c5fb82dd0069173, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/d30d64d4a2fc462aa9c0cd734da62b85] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp, totalSize=173.7 K 2024-12-06T16:01:25,512 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:25,512 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. files: [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/3d7cf3b8e68249449f5ebd0d75b6f59e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/e42c7f0fd4494750a8cc6b0ef3cb276b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/a2d4a2794538445a9c5fb82dd0069173, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/d30d64d4a2fc462aa9c0cd734da62b85] 2024-12-06T16:01:25,513 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d7cf3b8e68249449f5ebd0d75b6f59e, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733500881598 2024-12-06T16:01:25,513 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting e42c7f0fd4494750a8cc6b0ef3cb276b, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733500882753 2024-12-06T16:01:25,514 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2d4a2794538445a9c5fb82dd0069173, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733500883260 2024-12-06T16:01:25,514 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting d30d64d4a2fc462aa9c0cd734da62b85, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733500883937 2024-12-06T16:01:25,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:25,514 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T16:01:25,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d7db0f41a15490076613839fa83ecf9:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:01:25,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:25,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d7db0f41a15490076613839fa83ecf9:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:01:25,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:25,516 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T16:01:25,516 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 0d7db0f41a15490076613839fa83ecf9/B is initiating minor compaction (all files) 2024-12-06T16:01:25,517 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d7db0f41a15490076613839fa83ecf9/B in TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:25,517 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/9e4d971794e74f05bedcb4a33904182c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/11fcccfb86e04c1ebfd4e98349aaa86d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/272772f1e8904cbba890871cc952cd08, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/5f71aa6bfb00422a9c855ff5169ce48c] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp, totalSize=48.7 K 2024-12-06T16:01:25,517 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e4d971794e74f05bedcb4a33904182c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733500881598 2024-12-06T16:01:25,518 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 11fcccfb86e04c1ebfd4e98349aaa86d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733500883121 2024-12-06T16:01:25,518 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 272772f1e8904cbba890871cc952cd08, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733500883260 2024-12-06T16:01:25,519 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f71aa6bfb00422a9c855ff5169ce48c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733500883937 2024-12-06T16:01:25,533 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:25,542 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d7db0f41a15490076613839fa83ecf9#B#compaction#193 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:25,543 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/4f5262eb2e694e98baa26aa1ad0cf426 is 50, key is test_row_0/B:col10/1733500883937/Put/seqid=0 2024-12-06T16:01:25,547 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412066d85a3f497924cdaaca8c99dc8546fd2_0d7db0f41a15490076613839fa83ecf9 store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:25,553 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412066d85a3f497924cdaaca8c99dc8546fd2_0d7db0f41a15490076613839fa83ecf9, store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:25,554 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412066d85a3f497924cdaaca8c99dc8546fd2_0d7db0f41a15490076613839fa83ecf9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:25,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742055_1231 (size=13085) 2024-12-06T16:01:25,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-06T16:01:25,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-06T16:01:25,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:25,582 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 0d7db0f41a15490076613839fa83ecf9 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-06T16:01:25,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206243b47a72ae04c47899ef55b261170f1_0d7db0f41a15490076613839fa83ecf9 is 50, key is test_row_0/A:col10/1733500884640/Put/seqid=0 2024-12-06T16:01:25,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742056_1232 (size=4469) 2024-12-06T16:01:25,628 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d7db0f41a15490076613839fa83ecf9#A#compaction#192 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:25,628 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/25249d2fe679427b8158c55f0157c1ba is 175, key is test_row_0/A:col10/1733500883937/Put/seqid=0 2024-12-06T16:01:25,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742057_1233 (size=12454) 2024-12-06T16:01:25,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:25,653 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206243b47a72ae04c47899ef55b261170f1_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206243b47a72ae04c47899ef55b261170f1_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:25,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/6150b32a1ea245dea769b7b6e91121d0, store: [table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:25,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/6150b32a1ea245dea769b7b6e91121d0 is 175, key is test_row_0/A:col10/1733500884640/Put/seqid=0 2024-12-06T16:01:25,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742058_1234 (size=32039) 2024-12-06T16:01:25,702 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/25249d2fe679427b8158c55f0157c1ba as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/25249d2fe679427b8158c55f0157c1ba 2024-12-06T16:01:25,709 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0d7db0f41a15490076613839fa83ecf9/A of 0d7db0f41a15490076613839fa83ecf9 into 25249d2fe679427b8158c55f0157c1ba(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:25,709 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:25,709 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., storeName=0d7db0f41a15490076613839fa83ecf9/A, priority=12, startTime=1733500885508; duration=0sec 2024-12-06T16:01:25,710 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:25,710 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d7db0f41a15490076613839fa83ecf9:A 2024-12-06T16:01:25,710 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T16:01:25,712 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T16:01:25,712 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 0d7db0f41a15490076613839fa83ecf9/C is initiating minor compaction (all files) 2024-12-06T16:01:25,712 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d7db0f41a15490076613839fa83ecf9/C in TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:25,712 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/6b070626adf84ee4b4d5ad595bc1ef96, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/456dd17bb2d842c2b4a0c0af1f4b5003, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/05283e34e4c1425f8299e7a4732245a9, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/580cd27d4ee54406af0f618f398665ca] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp, totalSize=48.7 K 2024-12-06T16:01:25,713 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b070626adf84ee4b4d5ad595bc1ef96, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733500881598 2024-12-06T16:01:25,713 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 456dd17bb2d842c2b4a0c0af1f4b5003, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733500883121 2024-12-06T16:01:25,713 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05283e34e4c1425f8299e7a4732245a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733500883260 2024-12-06T16:01:25,714 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 580cd27d4ee54406af0f618f398665ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733500883937 2024-12-06T16:01:25,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742059_1235 (size=31255) 2024-12-06T16:01:25,729 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d7db0f41a15490076613839fa83ecf9#C#compaction#195 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:25,730 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/c5e180e1461b4e35b525fe120720dcc0 is 50, key is test_row_0/C:col10/1733500883937/Put/seqid=0 2024-12-06T16:01:25,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742060_1236 (size=13085) 2024-12-06T16:01:25,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:25,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:25,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:25,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500945803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:25,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:25,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500945807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:25,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:25,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:25,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500945809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:25,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:25,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:25,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:25,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:25,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:25,827 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-06T16:01:25,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-06T16:01:25,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:25,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500945912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:25,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:25,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:25,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:25,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:25,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500945922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:25,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:25,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:25,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500945923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:25,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:25,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:25,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:25,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:25,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:25,985 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/4f5262eb2e694e98baa26aa1ad0cf426 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/4f5262eb2e694e98baa26aa1ad0cf426 2024-12-06T16:01:25,992 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0d7db0f41a15490076613839fa83ecf9/B of 0d7db0f41a15490076613839fa83ecf9 into 4f5262eb2e694e98baa26aa1ad0cf426(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:25,992 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:25,992 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., storeName=0d7db0f41a15490076613839fa83ecf9/B, priority=12, startTime=1733500885514; duration=0sec 2024-12-06T16:01:25,992 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:25,992 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d7db0f41a15490076613839fa83ecf9:B 2024-12-06T16:01:26,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500946122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:26,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:26,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:26,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:26,129 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=362, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/6150b32a1ea245dea769b7b6e91121d0 2024-12-06T16:01:26,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500946131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:26,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/e84fb13caae44185a605c642f1a98307 is 50, key is test_row_0/B:col10/1733500884640/Put/seqid=0 2024-12-06T16:01:26,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500946132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:26,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:26,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:26,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:26,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:26,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:26,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:26,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742061_1237 (size=12301) 2024-12-06T16:01:26,154 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/e84fb13caae44185a605c642f1a98307 2024-12-06T16:01:26,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/5e0b838e91f944328d7c32e78adf3d5c is 50, key is test_row_0/C:col10/1733500884640/Put/seqid=0 2024-12-06T16:01:26,177 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/c5e180e1461b4e35b525fe120720dcc0 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/c5e180e1461b4e35b525fe120720dcc0 2024-12-06T16:01:26,184 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0d7db0f41a15490076613839fa83ecf9/C of 0d7db0f41a15490076613839fa83ecf9 into c5e180e1461b4e35b525fe120720dcc0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:26,184 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:26,184 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., storeName=0d7db0f41a15490076613839fa83ecf9/C, priority=12, startTime=1733500885515; duration=0sec 2024-12-06T16:01:26,184 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:26,184 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d7db0f41a15490076613839fa83ecf9:C 2024-12-06T16:01:26,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742062_1238 (size=12301) 2024-12-06T16:01:26,195 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/5e0b838e91f944328d7c32e78adf3d5c 2024-12-06T16:01:26,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/6150b32a1ea245dea769b7b6e91121d0 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/6150b32a1ea245dea769b7b6e91121d0 2024-12-06T16:01:26,207 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/6150b32a1ea245dea769b7b6e91121d0, entries=150, sequenceid=362, filesize=30.5 K 2024-12-06T16:01:26,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/e84fb13caae44185a605c642f1a98307 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/e84fb13caae44185a605c642f1a98307 2024-12-06T16:01:26,213 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/e84fb13caae44185a605c642f1a98307, entries=150, sequenceid=362, filesize=12.0 K 2024-12-06T16:01:26,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/5e0b838e91f944328d7c32e78adf3d5c as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/5e0b838e91f944328d7c32e78adf3d5c 2024-12-06T16:01:26,220 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/5e0b838e91f944328d7c32e78adf3d5c, entries=150, sequenceid=362, filesize=12.0 K 2024-12-06T16:01:26,221 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 0d7db0f41a15490076613839fa83ecf9 in 639ms, sequenceid=362, compaction requested=false 2024-12-06T16:01:26,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:26,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:26,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-06T16:01:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-06T16:01:26,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-06T16:01:26,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 952 msec 2024-12-06T16:01:26,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 962 msec 2024-12-06T16:01:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-06T16:01:26,402 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:01:26,403 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:01:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-06T16:01:26,406 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:01:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-06T16:01:26,407 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:01:26,407 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:01:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:26,435 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:01:26,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d7db0f41a15490076613839fa83ecf9 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-06T16:01:26,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206c328b3212af94e618c84597468e3d14a_0d7db0f41a15490076613839fa83ecf9 is 50, key is test_row_0/A:col10/1733500885796/Put/seqid=0 2024-12-06T16:01:26,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742063_1239 (size=14994) 2024-12-06T16:01:26,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500946483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:26,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:26,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:26,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:26,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500946485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:26,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:26,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:26,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500946486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:26,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:26,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:26,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:26,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-06T16:01:26,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-06T16:01:26,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:26,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:26,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:26,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:26,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:26,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500946591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:26,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:26,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:26,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:26,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:26,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500946601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:26,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500946602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:26,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:26,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:26,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:26,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:26,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:26,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:26,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-06T16:01:26,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:26,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:26,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:26,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:26,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:26,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-06T16:01:26,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500946801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:26,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:26,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:26,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:26,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:26,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500946812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:26,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:26,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:26,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:26,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500946814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:26,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:26,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:26,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:26,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-06T16:01:26,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:26,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:26,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:26,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:26,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:26,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:26,872 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:26,877 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206c328b3212af94e618c84597468e3d14a_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206c328b3212af94e618c84597468e3d14a_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:26,878 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/3cb205ebe0d34034a3b5d33569e9e7a0, store: [table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:26,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/3cb205ebe0d34034a3b5d33569e9e7a0 is 175, key is test_row_0/A:col10/1733500885796/Put/seqid=0 2024-12-06T16:01:26,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742064_1240 (size=39949) 2024-12-06T16:01:27,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-06T16:01:27,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:27,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:27,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:27,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:27,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-06T16:01:27,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500947112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:27,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500947122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:27,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500947122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:27,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:27,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:27,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:27,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:27,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:27,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:27,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-06T16:01:27,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:27,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:27,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:27,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:27,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:27,288 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=378, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/3cb205ebe0d34034a3b5d33569e9e7a0 2024-12-06T16:01:27,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/4a10d33460c2445faea4a26a9a808050 is 50, key is test_row_0/B:col10/1733500885796/Put/seqid=0 2024-12-06T16:01:27,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742065_1241 (size=12301) 2024-12-06T16:01:27,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/4a10d33460c2445faea4a26a9a808050 2024-12-06T16:01:27,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/6d2a253367744020985da379d8022c05 is 50, key is test_row_0/C:col10/1733500885796/Put/seqid=0 2024-12-06T16:01:27,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742066_1242 (size=12301) 2024-12-06T16:01:27,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/6d2a253367744020985da379d8022c05 2024-12-06T16:01:27,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/3cb205ebe0d34034a3b5d33569e9e7a0 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/3cb205ebe0d34034a3b5d33569e9e7a0 2024-12-06T16:01:27,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-06T16:01:27,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:27,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:27,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:27,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:27,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:27,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/3cb205ebe0d34034a3b5d33569e9e7a0, entries=200, sequenceid=378, filesize=39.0 K 2024-12-06T16:01:27,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/4a10d33460c2445faea4a26a9a808050 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/4a10d33460c2445faea4a26a9a808050 2024-12-06T16:01:27,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/4a10d33460c2445faea4a26a9a808050, entries=150, sequenceid=378, filesize=12.0 K 2024-12-06T16:01:27,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/6d2a253367744020985da379d8022c05 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/6d2a253367744020985da379d8022c05 2024-12-06T16:01:27,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/6d2a253367744020985da379d8022c05, entries=150, sequenceid=378, filesize=12.0 K 2024-12-06T16:01:27,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 0d7db0f41a15490076613839fa83ecf9 in 923ms, sequenceid=378, compaction requested=true 2024-12-06T16:01:27,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:27,359 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:27,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d7db0f41a15490076613839fa83ecf9:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:01:27,360 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:27,360 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 0d7db0f41a15490076613839fa83ecf9/A is initiating minor compaction (all files) 2024-12-06T16:01:27,360 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d7db0f41a15490076613839fa83ecf9/A in TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:27,360 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/25249d2fe679427b8158c55f0157c1ba, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/6150b32a1ea245dea769b7b6e91121d0, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/3cb205ebe0d34034a3b5d33569e9e7a0] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp, totalSize=100.8 K 2024-12-06T16:01:27,360 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:27,360 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. files: [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/25249d2fe679427b8158c55f0157c1ba, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/6150b32a1ea245dea769b7b6e91121d0, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/3cb205ebe0d34034a3b5d33569e9e7a0] 2024-12-06T16:01:27,360 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25249d2fe679427b8158c55f0157c1ba, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733500883937 2024-12-06T16:01:27,361 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6150b32a1ea245dea769b7b6e91121d0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1733500884585 2024-12-06T16:01:27,361 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cb205ebe0d34034a3b5d33569e9e7a0, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1733500885796 2024-12-06T16:01:27,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:27,362 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:27,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d7db0f41a15490076613839fa83ecf9:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:01:27,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:27,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d7db0f41a15490076613839fa83ecf9:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:01:27,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:27,366 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:27,366 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 0d7db0f41a15490076613839fa83ecf9/B is initiating minor compaction (all files) 2024-12-06T16:01:27,367 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d7db0f41a15490076613839fa83ecf9/B in TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:27,367 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/4f5262eb2e694e98baa26aa1ad0cf426, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/e84fb13caae44185a605c642f1a98307, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/4a10d33460c2445faea4a26a9a808050] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp, totalSize=36.8 K 2024-12-06T16:01:27,367 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f5262eb2e694e98baa26aa1ad0cf426, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733500883937 2024-12-06T16:01:27,368 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting e84fb13caae44185a605c642f1a98307, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1733500884585 2024-12-06T16:01:27,369 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a10d33460c2445faea4a26a9a808050, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1733500885796 2024-12-06T16:01:27,373 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:27,390 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206fcaa4498742d4dee9784857c1166a295_0d7db0f41a15490076613839fa83ecf9 store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:27,392 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206fcaa4498742d4dee9784857c1166a295_0d7db0f41a15490076613839fa83ecf9, store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:27,392 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206fcaa4498742d4dee9784857c1166a295_0d7db0f41a15490076613839fa83ecf9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:27,395 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d7db0f41a15490076613839fa83ecf9#B#compaction#202 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:27,396 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/cd1c543169084d5fa7cfb5a705a40f6b is 50, key is test_row_0/B:col10/1733500885796/Put/seqid=0 2024-12-06T16:01:27,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742067_1243 (size=4469) 2024-12-06T16:01:27,438 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d7db0f41a15490076613839fa83ecf9#A#compaction#201 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:27,438 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/af4aef5ae9464e26b37fc9ca42396e63 is 175, key is test_row_0/A:col10/1733500885796/Put/seqid=0 2024-12-06T16:01:27,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742068_1244 (size=13187) 2024-12-06T16:01:27,461 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/cd1c543169084d5fa7cfb5a705a40f6b as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/cd1c543169084d5fa7cfb5a705a40f6b 2024-12-06T16:01:27,469 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d7db0f41a15490076613839fa83ecf9/B of 0d7db0f41a15490076613839fa83ecf9 into cd1c543169084d5fa7cfb5a705a40f6b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:27,469 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:27,469 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., storeName=0d7db0f41a15490076613839fa83ecf9/B, priority=13, startTime=1733500887362; duration=0sec 2024-12-06T16:01:27,469 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:27,469 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d7db0f41a15490076613839fa83ecf9:B 2024-12-06T16:01:27,470 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:27,471 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:27,471 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 0d7db0f41a15490076613839fa83ecf9/C is initiating minor compaction (all files) 2024-12-06T16:01:27,471 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d7db0f41a15490076613839fa83ecf9/C in TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:27,471 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/c5e180e1461b4e35b525fe120720dcc0, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/5e0b838e91f944328d7c32e78adf3d5c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/6d2a253367744020985da379d8022c05] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp, totalSize=36.8 K 2024-12-06T16:01:27,471 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting c5e180e1461b4e35b525fe120720dcc0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733500883937 2024-12-06T16:01:27,472 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e0b838e91f944328d7c32e78adf3d5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1733500884585 2024-12-06T16:01:27,472 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d2a253367744020985da379d8022c05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1733500885796 2024-12-06T16:01:27,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-06T16:01:27,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:27,492 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 0d7db0f41a15490076613839fa83ecf9 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-06T16:01:27,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742069_1245 (size=32141) 2024-12-06T16:01:27,504 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d7db0f41a15490076613839fa83ecf9#C#compaction#203 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:27,505 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/364c42a21ddb494db9401b68a966e4bc is 50, key is test_row_0/C:col10/1733500885796/Put/seqid=0 2024-12-06T16:01:27,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120672ce347ac57a4476936bdfd5028e325e_0d7db0f41a15490076613839fa83ecf9 is 50, key is test_row_0/A:col10/1733500886484/Put/seqid=0 2024-12-06T16:01:27,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-06T16:01:27,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742070_1246 (size=13187) 2024-12-06T16:01:27,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742071_1247 (size=12454) 2024-12-06T16:01:27,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:27,556 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120672ce347ac57a4476936bdfd5028e325e_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120672ce347ac57a4476936bdfd5028e325e_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:27,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/1bb296f91f5f4a9ca1360d516d6ee910, store: [table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:27,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/1bb296f91f5f4a9ca1360d516d6ee910 is 175, key is test_row_0/A:col10/1733500886484/Put/seqid=0 2024-12-06T16:01:27,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742072_1248 (size=31255) 2024-12-06T16:01:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:27,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. as already flushing 2024-12-06T16:01:27,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:27,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500947643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:27,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500947646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:27,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500947651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:27,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:27,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:27,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:27,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:27,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:27,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:27,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:27,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:27,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:27,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:27,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500947762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:27,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:27,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500947762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:27,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:27,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:27,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:27,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:27,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500947762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:27,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:27,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:27,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:27,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:27,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:27,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:27,909 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/af4aef5ae9464e26b37fc9ca42396e63 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/af4aef5ae9464e26b37fc9ca42396e63 2024-12-06T16:01:27,917 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d7db0f41a15490076613839fa83ecf9/A of 0d7db0f41a15490076613839fa83ecf9 into af4aef5ae9464e26b37fc9ca42396e63(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:27,917 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:27,917 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., storeName=0d7db0f41a15490076613839fa83ecf9/A, priority=13, startTime=1733500887359; duration=0sec 2024-12-06T16:01:27,917 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:27,917 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d7db0f41a15490076613839fa83ecf9:A 2024-12-06T16:01:27,954 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/364c42a21ddb494db9401b68a966e4bc as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/364c42a21ddb494db9401b68a966e4bc 2024-12-06T16:01:27,961 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d7db0f41a15490076613839fa83ecf9/C of 0d7db0f41a15490076613839fa83ecf9 into 364c42a21ddb494db9401b68a966e4bc(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:27,961 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:27,961 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., storeName=0d7db0f41a15490076613839fa83ecf9/C, priority=13, startTime=1733500887363; duration=0sec 2024-12-06T16:01:27,962 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:27,962 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d7db0f41a15490076613839fa83ecf9:C 2024-12-06T16:01:27,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60240 deadline: 1733500947972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:27,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60288 deadline: 1733500947972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:27,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60262 deadline: 1733500947972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:01:27,980 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=402, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/1bb296f91f5f4a9ca1360d516d6ee910 2024-12-06T16:01:27,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/01f3abeca23045d895960c759502e6d8 is 50, key is test_row_0/B:col10/1733500886484/Put/seqid=0 2024-12-06T16:01:27,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742073_1249 (size=12301) 2024-12-06T16:01:27,999 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/01f3abeca23045d895960c759502e6d8 2024-12-06T16:01:28,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/ea2156050a874c478e5df38fbf215062 is 50, key is test_row_0/C:col10/1733500886484/Put/seqid=0 2024-12-06T16:01:28,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742074_1250 (size=12301) 2024-12-06T16:01:28,038 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/ea2156050a874c478e5df38fbf215062 2024-12-06T16:01:28,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/1bb296f91f5f4a9ca1360d516d6ee910 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/1bb296f91f5f4a9ca1360d516d6ee910 2024-12-06T16:01:28,050 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/1bb296f91f5f4a9ca1360d516d6ee910, entries=150, sequenceid=402, filesize=30.5 K 2024-12-06T16:01:28,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/01f3abeca23045d895960c759502e6d8 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/01f3abeca23045d895960c759502e6d8 2024-12-06T16:01:28,056 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/01f3abeca23045d895960c759502e6d8, entries=150, sequenceid=402, filesize=12.0 K 2024-12-06T16:01:28,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/ea2156050a874c478e5df38fbf215062 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/ea2156050a874c478e5df38fbf215062 2024-12-06T16:01:28,067 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/ea2156050a874c478e5df38fbf215062, entries=150, sequenceid=402, filesize=12.0 K 2024-12-06T16:01:28,068 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 0d7db0f41a15490076613839fa83ecf9 in 576ms, sequenceid=402, compaction requested=false 2024-12-06T16:01:28,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:28,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:28,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-06T16:01:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-06T16:01:28,073 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-06T16:01:28,073 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6630 sec 2024-12-06T16:01:28,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.6700 sec 2024-12-06T16:01:28,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:28,284 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:01:28,284 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d7db0f41a15490076613839fa83ecf9 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-06T16:01:28,294 INFO [Thread-692 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-692. 2024-12-06T16:01:28,294 INFO [Thread-694 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-694. 2024-12-06T16:01:28,294 DEBUG [Thread-692 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:01:28,294 DEBUG [Thread-694 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:01:28,294 DEBUG [Thread-692 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:28,294 DEBUG [Thread-694 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:28,294 DEBUG [Thread-692 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:28,294 DEBUG [Thread-694 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:28,294 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:01:28,294 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:01:28,297 INFO [Thread-703 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-703. 2024-12-06T16:01:28,297 DEBUG [Thread-703 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:01:28,297 DEBUG [Thread-703 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:28,297 INFO [Thread-699 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-699. 2024-12-06T16:01:28,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206b86c433afa0e461eafbaa9a891c97ff9_0d7db0f41a15490076613839fa83ecf9 is 50, key is test_row_0/A:col10/1733500887639/Put/seqid=0 2024-12-06T16:01:28,297 DEBUG [Thread-699 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:01:28,297 DEBUG [Thread-699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:28,297 DEBUG [Thread-703 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:28,297 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:01:28,298 DEBUG [Thread-699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:28,298 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:01:28,298 INFO [Thread-697 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-697. 2024-12-06T16:01:28,298 DEBUG [Thread-697 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:01:28,298 DEBUG [Thread-697 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:28,299 DEBUG [Thread-697 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:28,299 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:01:28,300 INFO [Thread-688 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-688. 2024-12-06T16:01:28,300 DEBUG [Thread-688 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:01:28,300 DEBUG [Thread-688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:28,300 DEBUG [Thread-688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:28,300 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:01:28,304 INFO [Thread-701 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-701. 2024-12-06T16:01:28,304 DEBUG [Thread-701 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:01:28,304 DEBUG [Thread-701 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:28,305 DEBUG [Thread-701 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:28,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742075_1251 (size=17534) 2024-12-06T16:01:28,305 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:01:28,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-06T16:01:28,552 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:01:28,705 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:28,710 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206b86c433afa0e461eafbaa9a891c97ff9_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206b86c433afa0e461eafbaa9a891c97ff9_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:28,711 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/345b1ad106b043358615394faf3b0f2e, store: [table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:28,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/345b1ad106b043358615394faf3b0f2e is 175, key is test_row_0/A:col10/1733500887639/Put/seqid=0 2024-12-06T16:01:28,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742076_1252 (size=48639) 2024-12-06T16:01:29,118 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=419, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/345b1ad106b043358615394faf3b0f2e 2024-12-06T16:01:29,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/171a2a725d3a443d8b929d2db4f587a8 is 50, key is test_row_0/B:col10/1733500887639/Put/seqid=0 2024-12-06T16:01:29,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742077_1253 (size=12301) 2024-12-06T16:01:29,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/171a2a725d3a443d8b929d2db4f587a8 2024-12-06T16:01:29,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/0b3a26cac196435ca41a3da56cb8e936 is 50, key is test_row_0/C:col10/1733500887639/Put/seqid=0 2024-12-06T16:01:29,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742078_1254 (size=12301) 2024-12-06T16:01:29,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/0b3a26cac196435ca41a3da56cb8e936 2024-12-06T16:01:29,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/345b1ad106b043358615394faf3b0f2e as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/345b1ad106b043358615394faf3b0f2e 2024-12-06T16:01:29,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/345b1ad106b043358615394faf3b0f2e, entries=250, sequenceid=419, filesize=47.5 K 2024-12-06T16:01:29,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/171a2a725d3a443d8b929d2db4f587a8 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/171a2a725d3a443d8b929d2db4f587a8 2024-12-06T16:01:29,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/171a2a725d3a443d8b929d2db4f587a8, entries=150, sequenceid=419, filesize=12.0 K 2024-12-06T16:01:29,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/0b3a26cac196435ca41a3da56cb8e936 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/0b3a26cac196435ca41a3da56cb8e936 2024-12-06T16:01:29,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/0b3a26cac196435ca41a3da56cb8e936, entries=150, sequenceid=419, filesize=12.0 K 2024-12-06T16:01:29,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=26.84 KB/27480 for 0d7db0f41a15490076613839fa83ecf9 in 1277ms, sequenceid=419, compaction requested=true 2024-12-06T16:01:29,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:29,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d7db0f41a15490076613839fa83ecf9:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:01:29,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:29,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d7db0f41a15490076613839fa83ecf9:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:01:29,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:29,561 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:29,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d7db0f41a15490076613839fa83ecf9:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:01:29,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:29,561 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:29,562 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 112035 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:29,562 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:29,562 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 0d7db0f41a15490076613839fa83ecf9/A is initiating minor compaction (all files) 2024-12-06T16:01:29,562 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 0d7db0f41a15490076613839fa83ecf9/B is initiating minor compaction (all files) 2024-12-06T16:01:29,562 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d7db0f41a15490076613839fa83ecf9/A in TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:29,562 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d7db0f41a15490076613839fa83ecf9/B in TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:29,562 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/af4aef5ae9464e26b37fc9ca42396e63, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/1bb296f91f5f4a9ca1360d516d6ee910, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/345b1ad106b043358615394faf3b0f2e] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp, totalSize=109.4 K 2024-12-06T16:01:29,562 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:29,563 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. files: [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/af4aef5ae9464e26b37fc9ca42396e63, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/1bb296f91f5f4a9ca1360d516d6ee910, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/345b1ad106b043358615394faf3b0f2e] 2024-12-06T16:01:29,563 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/cd1c543169084d5fa7cfb5a705a40f6b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/01f3abeca23045d895960c759502e6d8, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/171a2a725d3a443d8b929d2db4f587a8] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp, totalSize=36.9 K 2024-12-06T16:01:29,563 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting af4aef5ae9464e26b37fc9ca42396e63, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1733500885796 2024-12-06T16:01:29,563 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting cd1c543169084d5fa7cfb5a705a40f6b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1733500885796 2024-12-06T16:01:29,563 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bb296f91f5f4a9ca1360d516d6ee910, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733500886480 2024-12-06T16:01:29,563 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 01f3abeca23045d895960c759502e6d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733500886480 2024-12-06T16:01:29,564 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 345b1ad106b043358615394faf3b0f2e, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1733500887639 2024-12-06T16:01:29,564 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 171a2a725d3a443d8b929d2db4f587a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1733500887639 2024-12-06T16:01:29,573 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d7db0f41a15490076613839fa83ecf9#B#compaction#210 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:29,574 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:29,574 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/3d8ebae28b64431f89fee0977588163c is 50, key is test_row_0/B:col10/1733500887639/Put/seqid=0 2024-12-06T16:01:29,576 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412064e6d25ffaa9f44a5a8bafa9f130786a3_0d7db0f41a15490076613839fa83ecf9 store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:29,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742079_1255 (size=13289) 2024-12-06T16:01:29,606 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412064e6d25ffaa9f44a5a8bafa9f130786a3_0d7db0f41a15490076613839fa83ecf9, store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:29,606 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412064e6d25ffaa9f44a5a8bafa9f130786a3_0d7db0f41a15490076613839fa83ecf9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:29,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742080_1256 (size=4469) 2024-12-06T16:01:29,618 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d7db0f41a15490076613839fa83ecf9#A#compaction#211 average throughput is 0.56 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:29,619 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/360c498754724d4a923acce4418f41e3 is 175, key is test_row_0/A:col10/1733500887639/Put/seqid=0 2024-12-06T16:01:29,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742081_1257 (size=32243) 2024-12-06T16:01:29,640 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/360c498754724d4a923acce4418f41e3 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/360c498754724d4a923acce4418f41e3 2024-12-06T16:01:29,645 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d7db0f41a15490076613839fa83ecf9/A of 0d7db0f41a15490076613839fa83ecf9 into 360c498754724d4a923acce4418f41e3(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:29,645 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:29,646 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., storeName=0d7db0f41a15490076613839fa83ecf9/A, priority=13, startTime=1733500889561; duration=0sec 2024-12-06T16:01:29,646 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:29,646 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d7db0f41a15490076613839fa83ecf9:A 2024-12-06T16:01:29,646 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:29,647 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:29,647 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 0d7db0f41a15490076613839fa83ecf9/C is initiating minor compaction (all files) 2024-12-06T16:01:29,647 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d7db0f41a15490076613839fa83ecf9/C in TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:29,647 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/364c42a21ddb494db9401b68a966e4bc, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/ea2156050a874c478e5df38fbf215062, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/0b3a26cac196435ca41a3da56cb8e936] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp, totalSize=36.9 K 2024-12-06T16:01:29,647 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 364c42a21ddb494db9401b68a966e4bc, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1733500885796 2024-12-06T16:01:29,647 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea2156050a874c478e5df38fbf215062, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733500886480 2024-12-06T16:01:29,648 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b3a26cac196435ca41a3da56cb8e936, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1733500887639 2024-12-06T16:01:29,658 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d7db0f41a15490076613839fa83ecf9#C#compaction#212 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:29,658 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/d2cd6d9af721411eb9e3fcce73ec63ba is 50, key is test_row_0/C:col10/1733500887639/Put/seqid=0 2024-12-06T16:01:29,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742082_1258 (size=13289) 2024-12-06T16:01:29,987 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/3d8ebae28b64431f89fee0977588163c as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/3d8ebae28b64431f89fee0977588163c 2024-12-06T16:01:29,992 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d7db0f41a15490076613839fa83ecf9/B of 0d7db0f41a15490076613839fa83ecf9 into 3d8ebae28b64431f89fee0977588163c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:29,992 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:29,992 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., storeName=0d7db0f41a15490076613839fa83ecf9/B, priority=13, startTime=1733500889561; duration=0sec 2024-12-06T16:01:29,992 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:29,992 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d7db0f41a15490076613839fa83ecf9:B 2024-12-06T16:01:30,075 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/d2cd6d9af721411eb9e3fcce73ec63ba as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/d2cd6d9af721411eb9e3fcce73ec63ba 2024-12-06T16:01:30,081 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d7db0f41a15490076613839fa83ecf9/C of 0d7db0f41a15490076613839fa83ecf9 into d2cd6d9af721411eb9e3fcce73ec63ba(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:30,081 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d7db0f41a15490076613839fa83ecf9: 2024-12-06T16:01:30,081 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., storeName=0d7db0f41a15490076613839fa83ecf9/C, priority=13, startTime=1733500889561; duration=0sec 2024-12-06T16:01:30,081 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:30,081 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d7db0f41a15490076613839fa83ecf9:C 2024-12-06T16:01:30,134 INFO [Thread-690 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-690. 2024-12-06T16:01:30,134 DEBUG [Thread-690 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:01:30,134 DEBUG [Thread-690 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:30,134 DEBUG [Thread-690 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:30,134 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:01:30,204 INFO [Thread-686 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-686. 2024-12-06T16:01:30,204 DEBUG [Thread-686 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:01:30,204 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:30,204 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:30,204 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:01:30,204 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-06T16:01:30,204 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-12-06T16:01:30,204 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-12-06T16:01:30,204 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-12-06T16:01:30,204 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 114 2024-12-06T16:01:30,204 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 106 2024-12-06T16:01:30,204 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-06T16:01:30,204 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2855 2024-12-06T16:01:30,204 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2805 2024-12-06T16:01:30,204 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-06T16:01:30,204 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1287 2024-12-06T16:01:30,204 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3860 rows 2024-12-06T16:01:30,204 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1285 2024-12-06T16:01:30,204 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3854 rows 2024-12-06T16:01:30,205 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T16:01:30,205 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T16:01:30,205 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:30,205 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:30,205 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:01:30,206 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-06T16:01:30,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-06T16:01:30,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-06T16:01:30,209 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733500890209"}]},"ts":"1733500890209"} 2024-12-06T16:01:30,210 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-06T16:01:30,210 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-06T16:01:30,211 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-06T16:01:30,212 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d7db0f41a15490076613839fa83ecf9, UNASSIGN}] 2024-12-06T16:01:30,213 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d7db0f41a15490076613839fa83ecf9, UNASSIGN 2024-12-06T16:01:30,214 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=0d7db0f41a15490076613839fa83ecf9, regionState=CLOSING, regionLocation=85bef17d9292,46105,1733500827732 2024-12-06T16:01:30,215 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d7db0f41a15490076613839fa83ecf9, UNASSIGN because future has completed 2024-12-06T16:01:30,215 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-06T16:01:30,215 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732}] 2024-12-06T16:01:30,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-06T16:01:30,368 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(122): Close 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:30,368 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-06T16:01:30,368 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1722): Closing 0d7db0f41a15490076613839fa83ecf9, disabling compactions & flushes 2024-12-06T16:01:30,368 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:30,368 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:30,368 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. after waiting 0 ms 2024-12-06T16:01:30,368 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:30,368 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(2902): Flushing 0d7db0f41a15490076613839fa83ecf9 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-12-06T16:01:30,376 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412063145dc4e6f5546c9bc609e0ec356095f_0d7db0f41a15490076613839fa83ecf9 is 50, key is test_row_0/A:col10/1733500888288/Put/seqid=0 2024-12-06T16:01:30,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742083_1259 (size=12454) 2024-12-06T16:01:30,380 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:30,385 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412063145dc4e6f5546c9bc609e0ec356095f_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412063145dc4e6f5546c9bc609e0ec356095f_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:30,386 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/39f3ddf00d2144988623b6fde890a065, store: [table=TestAcidGuarantees family=A region=0d7db0f41a15490076613839fa83ecf9] 2024-12-06T16:01:30,387 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/39f3ddf00d2144988623b6fde890a065 is 175, key is test_row_0/A:col10/1733500888288/Put/seqid=0 2024-12-06T16:01:30,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742084_1260 (size=31255) 2024-12-06T16:01:30,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-06T16:01:30,793 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=431, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/39f3ddf00d2144988623b6fde890a065 2024-12-06T16:01:30,801 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/23f077d70eec4f3aaa7aad508268eb9d is 50, key is test_row_0/B:col10/1733500888288/Put/seqid=0 2024-12-06T16:01:30,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742085_1261 (size=12301) 2024-12-06T16:01:30,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-06T16:01:31,205 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/23f077d70eec4f3aaa7aad508268eb9d 2024-12-06T16:01:31,213 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/fb236a5aae024b139784783b7ec0990c is 50, key is test_row_0/C:col10/1733500888288/Put/seqid=0 2024-12-06T16:01:31,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742086_1262 (size=12301) 2024-12-06T16:01:31,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-06T16:01:31,618 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/fb236a5aae024b139784783b7ec0990c 2024-12-06T16:01:31,623 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/A/39f3ddf00d2144988623b6fde890a065 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/39f3ddf00d2144988623b6fde890a065 2024-12-06T16:01:31,626 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/39f3ddf00d2144988623b6fde890a065, entries=150, sequenceid=431, filesize=30.5 K 2024-12-06T16:01:31,627 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/B/23f077d70eec4f3aaa7aad508268eb9d as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/23f077d70eec4f3aaa7aad508268eb9d 2024-12-06T16:01:31,631 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/23f077d70eec4f3aaa7aad508268eb9d, entries=150, sequenceid=431, filesize=12.0 K 2024-12-06T16:01:31,632 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/.tmp/C/fb236a5aae024b139784783b7ec0990c as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/fb236a5aae024b139784783b7ec0990c 2024-12-06T16:01:31,635 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/fb236a5aae024b139784783b7ec0990c, entries=150, sequenceid=431, filesize=12.0 K 2024-12-06T16:01:31,636 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=0 B/0 for 0d7db0f41a15490076613839fa83ecf9 in 1268ms, sequenceid=431, compaction requested=false 2024-12-06T16:01:31,637 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/c0b2e11112104a1a89d88e7d79cf6caf, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/c86903ceafdf479e828a8ae7a77acf3f, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/7acc16ea6dc7428ba361ca5b29b3c9d3, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/a7d54b9ca03c4f8cb26d43afdb992010, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/784ee92929cb4f199c969cb1c24027a2, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/408e769fa08c439ba5b8dbc1ffcc435c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/c769a6a7841a46778d88e6691dd2c563, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/a0e5c9dceb734e628ae9f90909e9cf9b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/d405fb48fcf14f6590bcb1600e752d87, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/5fe7830232724c0583a5f49314f525cc, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/7e7c56ae6d46487490ad1b767bc9502f, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/50f91a0b5b7143a0b39b07bf16c55f51, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/f4a8f96525e24ae7ace5fd93aac058a6, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/ba0a969c3af5432093f956d11d01664d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/dad50953b79947c9abd53321e5b1ade7, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/b59783c5d1f643eda31253faaa7b57b8, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/6099cde958d24df38024e5b59388e69b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/1ed9be18977c4b73963df2240761adbc, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/462f1aaa86bd402f8c8c830146768f77, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/3d7cf3b8e68249449f5ebd0d75b6f59e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/e42c7f0fd4494750a8cc6b0ef3cb276b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/a2d4a2794538445a9c5fb82dd0069173, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/d30d64d4a2fc462aa9c0cd734da62b85, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/25249d2fe679427b8158c55f0157c1ba, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/6150b32a1ea245dea769b7b6e91121d0, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/3cb205ebe0d34034a3b5d33569e9e7a0, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/af4aef5ae9464e26b37fc9ca42396e63, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/1bb296f91f5f4a9ca1360d516d6ee910, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/345b1ad106b043358615394faf3b0f2e] to archive 2024-12-06T16:01:31,638 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T16:01:31,639 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/c0b2e11112104a1a89d88e7d79cf6caf to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/c0b2e11112104a1a89d88e7d79cf6caf 2024-12-06T16:01:31,640 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/c86903ceafdf479e828a8ae7a77acf3f to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/c86903ceafdf479e828a8ae7a77acf3f 2024-12-06T16:01:31,641 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/7acc16ea6dc7428ba361ca5b29b3c9d3 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/7acc16ea6dc7428ba361ca5b29b3c9d3 2024-12-06T16:01:31,643 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/a7d54b9ca03c4f8cb26d43afdb992010 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/a7d54b9ca03c4f8cb26d43afdb992010 2024-12-06T16:01:31,645 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/784ee92929cb4f199c969cb1c24027a2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/784ee92929cb4f199c969cb1c24027a2 2024-12-06T16:01:31,646 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/408e769fa08c439ba5b8dbc1ffcc435c to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/408e769fa08c439ba5b8dbc1ffcc435c 2024-12-06T16:01:31,647 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/c769a6a7841a46778d88e6691dd2c563 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/c769a6a7841a46778d88e6691dd2c563 2024-12-06T16:01:31,649 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/a0e5c9dceb734e628ae9f90909e9cf9b to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/a0e5c9dceb734e628ae9f90909e9cf9b 2024-12-06T16:01:31,650 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/d405fb48fcf14f6590bcb1600e752d87 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/d405fb48fcf14f6590bcb1600e752d87 2024-12-06T16:01:31,651 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/5fe7830232724c0583a5f49314f525cc to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/5fe7830232724c0583a5f49314f525cc 2024-12-06T16:01:31,652 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/7e7c56ae6d46487490ad1b767bc9502f to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/7e7c56ae6d46487490ad1b767bc9502f 2024-12-06T16:01:31,654 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/50f91a0b5b7143a0b39b07bf16c55f51 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/50f91a0b5b7143a0b39b07bf16c55f51 2024-12-06T16:01:31,655 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/f4a8f96525e24ae7ace5fd93aac058a6 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/f4a8f96525e24ae7ace5fd93aac058a6 2024-12-06T16:01:31,657 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/ba0a969c3af5432093f956d11d01664d to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/ba0a969c3af5432093f956d11d01664d 2024-12-06T16:01:31,658 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/dad50953b79947c9abd53321e5b1ade7 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/dad50953b79947c9abd53321e5b1ade7 2024-12-06T16:01:31,659 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/b59783c5d1f643eda31253faaa7b57b8 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/b59783c5d1f643eda31253faaa7b57b8 2024-12-06T16:01:31,660 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/6099cde958d24df38024e5b59388e69b to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/6099cde958d24df38024e5b59388e69b 2024-12-06T16:01:31,661 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/1ed9be18977c4b73963df2240761adbc to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/1ed9be18977c4b73963df2240761adbc 2024-12-06T16:01:31,662 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/462f1aaa86bd402f8c8c830146768f77 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/462f1aaa86bd402f8c8c830146768f77 2024-12-06T16:01:31,664 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/3d7cf3b8e68249449f5ebd0d75b6f59e to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/3d7cf3b8e68249449f5ebd0d75b6f59e 2024-12-06T16:01:31,665 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/e42c7f0fd4494750a8cc6b0ef3cb276b to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/e42c7f0fd4494750a8cc6b0ef3cb276b 2024-12-06T16:01:31,666 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/a2d4a2794538445a9c5fb82dd0069173 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/a2d4a2794538445a9c5fb82dd0069173 2024-12-06T16:01:31,668 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/d30d64d4a2fc462aa9c0cd734da62b85 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/d30d64d4a2fc462aa9c0cd734da62b85 2024-12-06T16:01:31,669 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/25249d2fe679427b8158c55f0157c1ba to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/25249d2fe679427b8158c55f0157c1ba 2024-12-06T16:01:31,670 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/6150b32a1ea245dea769b7b6e91121d0 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/6150b32a1ea245dea769b7b6e91121d0 2024-12-06T16:01:31,671 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/3cb205ebe0d34034a3b5d33569e9e7a0 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/3cb205ebe0d34034a3b5d33569e9e7a0 2024-12-06T16:01:31,672 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/af4aef5ae9464e26b37fc9ca42396e63 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/af4aef5ae9464e26b37fc9ca42396e63 2024-12-06T16:01:31,673 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/1bb296f91f5f4a9ca1360d516d6ee910 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/1bb296f91f5f4a9ca1360d516d6ee910 2024-12-06T16:01:31,674 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/345b1ad106b043358615394faf3b0f2e to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/345b1ad106b043358615394faf3b0f2e 2024-12-06T16:01:31,676 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/d8891743d2f740ed97997b54eb9bea03, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/15d3261d28684bdebe3a33e7f9f42fc0, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/8f8d004ea376416ea789e469ad61acc7, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/10858766e5fd444e8f1cee1af0c16530, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/fd118a25ced4453ca94bf403306d550d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/4067755f0a174bf7987fe9d23a8d2d8f, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/432e292fc1e14b859acfe24780f3a518, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/4c4524a231af4ccebcea8e03afbf071b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/78c0b399d64742d197f2919007511ef2, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/2a54925edb164bb1b68ce1f4b9cb482a, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/98f2d8f4920e4d5987ce6f577885d916, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/ab160feb28594e1ea85b48e535980f80, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/8eab2601ac134edb8b5b99c074671e99, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/79d7c1cdc0834f0b97ceddaf3018f536, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/a16bd9eebef94a5eafaa843161b2f3f4, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/fb2e3329863f43589d2a12f5e5139905, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/1961566d2993404c8d2a969963088721, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/29a5cb3989544a04a6ef6329d1f8d551, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/9e4d971794e74f05bedcb4a33904182c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/ead077119c67435492b4adff00cb3bb9, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/11fcccfb86e04c1ebfd4e98349aaa86d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/272772f1e8904cbba890871cc952cd08, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/4f5262eb2e694e98baa26aa1ad0cf426, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/5f71aa6bfb00422a9c855ff5169ce48c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/e84fb13caae44185a605c642f1a98307, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/cd1c543169084d5fa7cfb5a705a40f6b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/4a10d33460c2445faea4a26a9a808050, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/01f3abeca23045d895960c759502e6d8, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/171a2a725d3a443d8b929d2db4f587a8] to archive 2024-12-06T16:01:31,677 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T16:01:31,678 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/d8891743d2f740ed97997b54eb9bea03 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/d8891743d2f740ed97997b54eb9bea03 2024-12-06T16:01:31,679 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/15d3261d28684bdebe3a33e7f9f42fc0 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/15d3261d28684bdebe3a33e7f9f42fc0 2024-12-06T16:01:31,681 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/8f8d004ea376416ea789e469ad61acc7 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/8f8d004ea376416ea789e469ad61acc7 2024-12-06T16:01:31,682 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/10858766e5fd444e8f1cee1af0c16530 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/10858766e5fd444e8f1cee1af0c16530 2024-12-06T16:01:31,683 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/fd118a25ced4453ca94bf403306d550d to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/fd118a25ced4453ca94bf403306d550d 2024-12-06T16:01:31,685 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/4067755f0a174bf7987fe9d23a8d2d8f to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/4067755f0a174bf7987fe9d23a8d2d8f 2024-12-06T16:01:31,686 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/432e292fc1e14b859acfe24780f3a518 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/432e292fc1e14b859acfe24780f3a518 2024-12-06T16:01:31,687 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/4c4524a231af4ccebcea8e03afbf071b to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/4c4524a231af4ccebcea8e03afbf071b 2024-12-06T16:01:31,688 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/78c0b399d64742d197f2919007511ef2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/78c0b399d64742d197f2919007511ef2 2024-12-06T16:01:31,689 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/2a54925edb164bb1b68ce1f4b9cb482a to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/2a54925edb164bb1b68ce1f4b9cb482a 2024-12-06T16:01:31,690 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/98f2d8f4920e4d5987ce6f577885d916 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/98f2d8f4920e4d5987ce6f577885d916 2024-12-06T16:01:31,691 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/ab160feb28594e1ea85b48e535980f80 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/ab160feb28594e1ea85b48e535980f80 2024-12-06T16:01:31,692 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/8eab2601ac134edb8b5b99c074671e99 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/8eab2601ac134edb8b5b99c074671e99 2024-12-06T16:01:31,693 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/79d7c1cdc0834f0b97ceddaf3018f536 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/79d7c1cdc0834f0b97ceddaf3018f536 2024-12-06T16:01:31,694 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/a16bd9eebef94a5eafaa843161b2f3f4 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/a16bd9eebef94a5eafaa843161b2f3f4 2024-12-06T16:01:31,695 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/fb2e3329863f43589d2a12f5e5139905 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/fb2e3329863f43589d2a12f5e5139905 2024-12-06T16:01:31,696 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/1961566d2993404c8d2a969963088721 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/1961566d2993404c8d2a969963088721 2024-12-06T16:01:31,697 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/29a5cb3989544a04a6ef6329d1f8d551 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/29a5cb3989544a04a6ef6329d1f8d551 2024-12-06T16:01:31,698 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/9e4d971794e74f05bedcb4a33904182c to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/9e4d971794e74f05bedcb4a33904182c 2024-12-06T16:01:31,699 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/ead077119c67435492b4adff00cb3bb9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/ead077119c67435492b4adff00cb3bb9 2024-12-06T16:01:31,700 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/11fcccfb86e04c1ebfd4e98349aaa86d to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/11fcccfb86e04c1ebfd4e98349aaa86d 2024-12-06T16:01:31,701 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/272772f1e8904cbba890871cc952cd08 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/272772f1e8904cbba890871cc952cd08 2024-12-06T16:01:31,702 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/4f5262eb2e694e98baa26aa1ad0cf426 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/4f5262eb2e694e98baa26aa1ad0cf426 2024-12-06T16:01:31,703 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/5f71aa6bfb00422a9c855ff5169ce48c to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/5f71aa6bfb00422a9c855ff5169ce48c 2024-12-06T16:01:31,704 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/e84fb13caae44185a605c642f1a98307 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/e84fb13caae44185a605c642f1a98307 2024-12-06T16:01:31,705 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/cd1c543169084d5fa7cfb5a705a40f6b to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/cd1c543169084d5fa7cfb5a705a40f6b 2024-12-06T16:01:31,706 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/4a10d33460c2445faea4a26a9a808050 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/4a10d33460c2445faea4a26a9a808050 2024-12-06T16:01:31,707 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/01f3abeca23045d895960c759502e6d8 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/01f3abeca23045d895960c759502e6d8 2024-12-06T16:01:31,708 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/171a2a725d3a443d8b929d2db4f587a8 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/171a2a725d3a443d8b929d2db4f587a8 2024-12-06T16:01:31,709 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/c5476b0259854be9864031ac442a32d2, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/f8e5cae6116447868200e9464483e999, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/52557655d8aa4afa82e74c90821660ae, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/e086a6cb71f84013841049eb5a4beaff, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/57da6b55f1fb42e8a1c9cca47303615b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/f4d8dde07e3448a09322ea1beffc2f0f, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/94b9cfe17d574644b9ea90313d728233, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/986e48a09c694262ba7228d89f90b3fd, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/0844460a4d0d4f5fb639d68bd8125ec6, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/5d70fe7d37af490e8bd513e3697c4dfb, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/a462be074bf84d7cb3e92b7ef3b92cce, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/c75ebfccc4ef4226addb1903ef797655, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/f0a1b6aa37a346c6a6d1e7ceffc01a63, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/2a355c02c5db47afb3ca192bf4cb7e1f, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/073363fb242549e1ab10b2e7b02cf9aa, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/42d19f131cb34425a222fb715cb2f01b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/777e05c1f7e24416ad60d2beccafad6c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/6c28b13afd564c39a89f13d5567ad7c1, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/6b070626adf84ee4b4d5ad595bc1ef96, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/43434bb12fb447df883391ebeef96c59, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/456dd17bb2d842c2b4a0c0af1f4b5003, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/05283e34e4c1425f8299e7a4732245a9, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/c5e180e1461b4e35b525fe120720dcc0, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/580cd27d4ee54406af0f618f398665ca, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/5e0b838e91f944328d7c32e78adf3d5c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/364c42a21ddb494db9401b68a966e4bc, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/6d2a253367744020985da379d8022c05, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/ea2156050a874c478e5df38fbf215062, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/0b3a26cac196435ca41a3da56cb8e936] to archive 2024-12-06T16:01:31,710 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T16:01:31,712 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/c5476b0259854be9864031ac442a32d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/c5476b0259854be9864031ac442a32d2 2024-12-06T16:01:31,713 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/f8e5cae6116447868200e9464483e999 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/f8e5cae6116447868200e9464483e999 2024-12-06T16:01:31,713 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/52557655d8aa4afa82e74c90821660ae to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/52557655d8aa4afa82e74c90821660ae 2024-12-06T16:01:31,714 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/e086a6cb71f84013841049eb5a4beaff to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/e086a6cb71f84013841049eb5a4beaff 2024-12-06T16:01:31,715 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/57da6b55f1fb42e8a1c9cca47303615b to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/57da6b55f1fb42e8a1c9cca47303615b 2024-12-06T16:01:31,717 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/f4d8dde07e3448a09322ea1beffc2f0f to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/f4d8dde07e3448a09322ea1beffc2f0f 2024-12-06T16:01:31,718 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/94b9cfe17d574644b9ea90313d728233 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/94b9cfe17d574644b9ea90313d728233 2024-12-06T16:01:31,719 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/986e48a09c694262ba7228d89f90b3fd to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/986e48a09c694262ba7228d89f90b3fd 2024-12-06T16:01:31,720 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/0844460a4d0d4f5fb639d68bd8125ec6 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/0844460a4d0d4f5fb639d68bd8125ec6 2024-12-06T16:01:31,722 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/5d70fe7d37af490e8bd513e3697c4dfb to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/5d70fe7d37af490e8bd513e3697c4dfb 2024-12-06T16:01:31,723 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/a462be074bf84d7cb3e92b7ef3b92cce to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/a462be074bf84d7cb3e92b7ef3b92cce 2024-12-06T16:01:31,724 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/c75ebfccc4ef4226addb1903ef797655 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/c75ebfccc4ef4226addb1903ef797655 2024-12-06T16:01:31,725 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/f0a1b6aa37a346c6a6d1e7ceffc01a63 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/f0a1b6aa37a346c6a6d1e7ceffc01a63 2024-12-06T16:01:31,726 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/2a355c02c5db47afb3ca192bf4cb7e1f to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/2a355c02c5db47afb3ca192bf4cb7e1f 2024-12-06T16:01:31,726 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/073363fb242549e1ab10b2e7b02cf9aa to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/073363fb242549e1ab10b2e7b02cf9aa 2024-12-06T16:01:31,727 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/42d19f131cb34425a222fb715cb2f01b to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/42d19f131cb34425a222fb715cb2f01b 2024-12-06T16:01:31,728 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/777e05c1f7e24416ad60d2beccafad6c to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/777e05c1f7e24416ad60d2beccafad6c 2024-12-06T16:01:31,730 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/6c28b13afd564c39a89f13d5567ad7c1 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/6c28b13afd564c39a89f13d5567ad7c1 2024-12-06T16:01:31,731 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/6b070626adf84ee4b4d5ad595bc1ef96 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/6b070626adf84ee4b4d5ad595bc1ef96 2024-12-06T16:01:31,733 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/43434bb12fb447df883391ebeef96c59 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/43434bb12fb447df883391ebeef96c59 2024-12-06T16:01:31,734 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/456dd17bb2d842c2b4a0c0af1f4b5003 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/456dd17bb2d842c2b4a0c0af1f4b5003 2024-12-06T16:01:31,735 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/05283e34e4c1425f8299e7a4732245a9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/05283e34e4c1425f8299e7a4732245a9 2024-12-06T16:01:31,736 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/c5e180e1461b4e35b525fe120720dcc0 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/c5e180e1461b4e35b525fe120720dcc0 2024-12-06T16:01:31,737 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/580cd27d4ee54406af0f618f398665ca to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/580cd27d4ee54406af0f618f398665ca 2024-12-06T16:01:31,738 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/5e0b838e91f944328d7c32e78adf3d5c to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/5e0b838e91f944328d7c32e78adf3d5c 2024-12-06T16:01:31,739 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/364c42a21ddb494db9401b68a966e4bc to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/364c42a21ddb494db9401b68a966e4bc 2024-12-06T16:01:31,740 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/6d2a253367744020985da379d8022c05 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/6d2a253367744020985da379d8022c05 2024-12-06T16:01:31,741 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/ea2156050a874c478e5df38fbf215062 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/ea2156050a874c478e5df38fbf215062 2024-12-06T16:01:31,742 DEBUG [StoreCloser-TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/0b3a26cac196435ca41a3da56cb8e936 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/0b3a26cac196435ca41a3da56cb8e936 2024-12-06T16:01:31,747 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/recovered.edits/434.seqid, newMaxSeqId=434, maxSeqId=4 2024-12-06T16:01:31,748 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. 2024-12-06T16:01:31,749 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1676): Region close journal for 0d7db0f41a15490076613839fa83ecf9: Waiting for close lock at 1733500890368Running coprocessor pre-close hooks at 1733500890368Disabling compacts and flushes for region at 1733500890368Disabling writes for close at 1733500890368Obtaining lock to block concurrent updates at 1733500890368Preparing flush snapshotting stores in 0d7db0f41a15490076613839fa83ecf9 at 1733500890368Finished memstore snapshotting TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9., syncing WAL and waiting on mvcc, flushsize=dataSize=41220, getHeapSize=108720, getOffHeapSize=0, getCellsCount=900 at 1733500890369 (+1 ms)Flushing stores of TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9. at 1733500890369Flushing 0d7db0f41a15490076613839fa83ecf9/A: creating writer at 1733500890369Flushing mob file 0d7db0f41a15490076613839fa83ecf9/A: appending metadata at 1733500890376 (+7 ms)Flushing mob file 0d7db0f41a15490076613839fa83ecf9/A: closing flushed file at 1733500890376Flushing 0d7db0f41a15490076613839fa83ecf9/A: appending metadata at 1733500890386 (+10 ms)Flushing 0d7db0f41a15490076613839fa83ecf9/A: closing flushed file at 1733500890387 (+1 ms)Flushing 0d7db0f41a15490076613839fa83ecf9/B: creating writer at 1733500890798 (+411 ms)Flushing 0d7db0f41a15490076613839fa83ecf9/B: appending metadata at 1733500890800 (+2 ms)Flushing 0d7db0f41a15490076613839fa83ecf9/B: closing flushed file at 1733500890800Flushing 0d7db0f41a15490076613839fa83ecf9/C: creating writer at 1733500891210 (+410 ms)Flushing 0d7db0f41a15490076613839fa83ecf9/C: appending metadata at 1733500891213 (+3 ms)Flushing 0d7db0f41a15490076613839fa83ecf9/C: closing flushed file at 1733500891213Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f8cea18: reopening flushed file at 1733500891622 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4fa79b46: reopening flushed file at 1733500891626 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a68aeb9: reopening flushed file at 1733500891631 (+5 ms)Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=0 B/0 for 0d7db0f41a15490076613839fa83ecf9 in 1268ms, sequenceid=431, compaction requested=false at 1733500891636 (+5 ms)Writing region close event to WAL at 1733500891744 (+108 ms)Running coprocessor post-close hooks at 1733500891748 (+4 ms)Closed at 1733500891748 2024-12-06T16:01:31,751 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(157): Closed 0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:31,751 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=0d7db0f41a15490076613839fa83ecf9, regionState=CLOSED 2024-12-06T16:01:31,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 because future has completed 2024-12-06T16:01:31,755 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-06T16:01:31,755 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseRegionProcedure 0d7db0f41a15490076613839fa83ecf9, server=85bef17d9292,46105,1733500827732 in 1.5390 sec 2024-12-06T16:01:31,757 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=51 2024-12-06T16:01:31,757 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=51, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d7db0f41a15490076613839fa83ecf9, UNASSIGN in 1.5430 sec 2024-12-06T16:01:31,759 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-06T16:01:31,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5470 sec 2024-12-06T16:01:31,761 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733500891761"}]},"ts":"1733500891761"} 2024-12-06T16:01:31,762 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-06T16:01:31,762 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-06T16:01:31,764 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5570 sec 2024-12-06T16:01:32,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-06T16:01:32,353 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:01:32,353 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-06T16:01:32,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:01:32,356 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:01:32,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-06T16:01:32,357 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=54, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:01:32,359 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:32,362 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A, FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B, FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C, FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/recovered.edits] 2024-12-06T16:01:32,365 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/360c498754724d4a923acce4418f41e3 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/360c498754724d4a923acce4418f41e3 2024-12-06T16:01:32,366 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/39f3ddf00d2144988623b6fde890a065 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/A/39f3ddf00d2144988623b6fde890a065 2024-12-06T16:01:32,368 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/23f077d70eec4f3aaa7aad508268eb9d to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/23f077d70eec4f3aaa7aad508268eb9d 2024-12-06T16:01:32,369 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/3d8ebae28b64431f89fee0977588163c to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/B/3d8ebae28b64431f89fee0977588163c 2024-12-06T16:01:32,372 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/d2cd6d9af721411eb9e3fcce73ec63ba to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/d2cd6d9af721411eb9e3fcce73ec63ba 2024-12-06T16:01:32,373 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/fb236a5aae024b139784783b7ec0990c to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/C/fb236a5aae024b139784783b7ec0990c 2024-12-06T16:01:32,375 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/recovered.edits/434.seqid to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9/recovered.edits/434.seqid 2024-12-06T16:01:32,376 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:32,376 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-06T16:01:32,376 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-06T16:01:32,377 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-06T16:01:32,381 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206243b47a72ae04c47899ef55b261170f1_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206243b47a72ae04c47899ef55b261170f1_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:32,382 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412063145dc4e6f5546c9bc609e0ec356095f_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412063145dc4e6f5546c9bc609e0ec356095f_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:32,383 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412063685169224cc40478718ccbc915f2a2a_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412063685169224cc40478718ccbc915f2a2a_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:32,384 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412064d6dd87f43b24a2cb7499bccdc772161_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412064d6dd87f43b24a2cb7499bccdc772161_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:32,385 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206509e3d48889a4ac4b88424131d3bcf8d_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206509e3d48889a4ac4b88424131d3bcf8d_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:32,387 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120672ce347ac57a4476936bdfd5028e325e_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120672ce347ac57a4476936bdfd5028e325e_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:32,388 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206806144f2f7424cf49e9fce14c9169510_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206806144f2f7424cf49e9fce14c9169510_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:32,389 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412068f04fb006e2c44f9b3b902a8d44be608_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412068f04fb006e2c44f9b3b902a8d44be608_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:32,390 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120690224034a8b64b0ea46afbfa0643f0d7_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120690224034a8b64b0ea46afbfa0643f0d7_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:32,391 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412069b53c6a980874f11a8bf680b97fe49bd_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412069b53c6a980874f11a8bf680b97fe49bd_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:32,392 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a976dab4463a4211bd3560ebd46c269d_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a976dab4463a4211bd3560ebd46c269d_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:32,393 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206b2bcd1c2a68b4e5ea20c4fe9d4bbdea8_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206b2bcd1c2a68b4e5ea20c4fe9d4bbdea8_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:32,394 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206b86c433afa0e461eafbaa9a891c97ff9_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206b86c433afa0e461eafbaa9a891c97ff9_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:32,396 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206bb3f6e75825b4d23872fd89fec8741e7_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206bb3f6e75825b4d23872fd89fec8741e7_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:32,397 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206be2e266758054a15a669b1832c61aa00_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206be2e266758054a15a669b1832c61aa00_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:32,398 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206c328b3212af94e618c84597468e3d14a_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206c328b3212af94e618c84597468e3d14a_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:32,399 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206cb9cd903a96a435bb6163f5d73fe5473_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206cb9cd903a96a435bb6163f5d73fe5473_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:32,400 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206d7274579658a40a9bc3304137d8972b6_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206d7274579658a40a9bc3304137d8972b6_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:32,401 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206d75c5957a69548dfbd5778787156bc26_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206d75c5957a69548dfbd5778787156bc26_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:32,402 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206dde3b68deda2443091ad6b021b783d9d_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206dde3b68deda2443091ad6b021b783d9d_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:32,403 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ddf4e89a12294b12a07275a9d35bcb12_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ddf4e89a12294b12a07275a9d35bcb12_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:32,404 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206f2818c888dfa4db988ee5532571d21d4_0d7db0f41a15490076613839fa83ecf9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206f2818c888dfa4db988ee5532571d21d4_0d7db0f41a15490076613839fa83ecf9 2024-12-06T16:01:32,405 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-06T16:01:32,407 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=54, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:01:32,409 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-06T16:01:32,411 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-06T16:01:32,412 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=54, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:01:32,412 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-06T16:01:32,412 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733500892412"}]},"ts":"9223372036854775807"} 2024-12-06T16:01:32,415 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-06T16:01:32,415 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0d7db0f41a15490076613839fa83ecf9, NAME => 'TestAcidGuarantees,,1733500866633.0d7db0f41a15490076613839fa83ecf9.', STARTKEY => '', ENDKEY => ''}] 2024-12-06T16:01:32,415 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-06T16:01:32,415 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733500892415"}]},"ts":"9223372036854775807"} 2024-12-06T16:01:32,417 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-06T16:01:32,417 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=54, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:01:32,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 64 msec 2024-12-06T16:01:32,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-06T16:01:32,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-06T16:01:32,462 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:01:32,476 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=242 (was 237) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/cluster_d00c8b32-e29e-fabf-e232-f9677adb6979/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_784179071_22 at /127.0.0.1:56156 [Waiting for operation #154] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_130634295_22 at /127.0.0.1:59570 [Waiting for operation #498] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_784179071_22 at /127.0.0.1:46954 [Waiting for operation #313] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_130634295_22 at /127.0.0.1:59660 [Waiting for operation #476] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/cluster_d00c8b32-e29e-fabf-e232-f9677adb6979/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=582 (was 494) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 12), AvailableMemoryMB=8764 (was 8875) 2024-12-06T16:01:32,485 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=242, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=582, ProcessCount=11, AvailableMemoryMB=8763 2024-12-06T16:01:32,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-06T16:01:32,486 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T16:01:32,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-06T16:01:32,489 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T16:01:32,489 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:32,489 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 55 2024-12-06T16:01:32,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-06T16:01:32,489 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T16:01:32,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742087_1263 (size=959) 2024-12-06T16:01:32,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-06T16:01:32,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-06T16:01:32,897 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d 2024-12-06T16:01:32,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742088_1264 (size=53) 2024-12-06T16:01:33,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-06T16:01:33,304 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T16:01:33,304 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 51b4277372e700eb3a1230f9ab87509e, disabling compactions & flushes 2024-12-06T16:01:33,304 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:33,304 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:33,304 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. after waiting 0 ms 2024-12-06T16:01:33,304 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:33,304 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:33,304 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 51b4277372e700eb3a1230f9ab87509e: Waiting for close lock at 1733500893304Disabling compacts and flushes for region at 1733500893304Disabling writes for close at 1733500893304Writing region close event to WAL at 1733500893304Closed at 1733500893304 2024-12-06T16:01:33,305 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T16:01:33,306 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733500893306"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733500893306"}]},"ts":"1733500893306"} 2024-12-06T16:01:33,307 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T16:01:33,308 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T16:01:33,309 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733500893308"}]},"ts":"1733500893308"} 2024-12-06T16:01:33,310 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-06T16:01:33,310 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=51b4277372e700eb3a1230f9ab87509e, ASSIGN}] 2024-12-06T16:01:33,311 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=51b4277372e700eb3a1230f9ab87509e, ASSIGN 2024-12-06T16:01:33,312 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=51b4277372e700eb3a1230f9ab87509e, ASSIGN; state=OFFLINE, location=85bef17d9292,46105,1733500827732; forceNewPlan=false, retain=false 2024-12-06T16:01:33,463 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=51b4277372e700eb3a1230f9ab87509e, regionState=OPENING, regionLocation=85bef17d9292,46105,1733500827732 2024-12-06T16:01:33,465 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=51b4277372e700eb3a1230f9ab87509e, ASSIGN because future has completed 2024-12-06T16:01:33,465 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure 51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732}] 2024-12-06T16:01:33,620 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:33,620 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7752): Opening region: {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} 2024-12-06T16:01:33,621 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:33,621 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T16:01:33,621 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7794): checking encryption for 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:33,621 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7797): checking classloading for 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:33,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-06T16:01:33,623 INFO [StoreOpener-51b4277372e700eb3a1230f9ab87509e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:33,624 INFO [StoreOpener-51b4277372e700eb3a1230f9ab87509e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51b4277372e700eb3a1230f9ab87509e columnFamilyName A 2024-12-06T16:01:33,624 DEBUG [StoreOpener-51b4277372e700eb3a1230f9ab87509e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:33,625 INFO [StoreOpener-51b4277372e700eb3a1230f9ab87509e-1 {}] regionserver.HStore(327): Store=51b4277372e700eb3a1230f9ab87509e/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T16:01:33,625 INFO [StoreOpener-51b4277372e700eb3a1230f9ab87509e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:33,626 INFO [StoreOpener-51b4277372e700eb3a1230f9ab87509e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51b4277372e700eb3a1230f9ab87509e columnFamilyName B 2024-12-06T16:01:33,626 DEBUG [StoreOpener-51b4277372e700eb3a1230f9ab87509e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:33,626 INFO [StoreOpener-51b4277372e700eb3a1230f9ab87509e-1 {}] regionserver.HStore(327): Store=51b4277372e700eb3a1230f9ab87509e/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T16:01:33,626 INFO [StoreOpener-51b4277372e700eb3a1230f9ab87509e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:33,627 INFO [StoreOpener-51b4277372e700eb3a1230f9ab87509e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51b4277372e700eb3a1230f9ab87509e columnFamilyName C 2024-12-06T16:01:33,628 DEBUG [StoreOpener-51b4277372e700eb3a1230f9ab87509e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:01:33,628 INFO [StoreOpener-51b4277372e700eb3a1230f9ab87509e-1 {}] regionserver.HStore(327): Store=51b4277372e700eb3a1230f9ab87509e/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T16:01:33,628 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1038): replaying wal for 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:33,629 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:33,629 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:33,630 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1048): stopping wal replay for 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:33,630 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1060): Cleaning up temporary data for 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:33,630 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T16:01:33,631 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1093): writing seq id for 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:33,633 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T16:01:33,633 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1114): Opened 51b4277372e700eb3a1230f9ab87509e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64879546, jitterRate=-0.03321942687034607}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T16:01:33,634 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:33,634 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1006): Region open journal for 51b4277372e700eb3a1230f9ab87509e: Running coprocessor pre-open hook at 1733500893621Writing region info on filesystem at 1733500893621Initializing all the Stores at 1733500893622 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733500893622Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733500893622Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733500893622Cleaning up temporary data from old regions at 1733500893630 (+8 ms)Running coprocessor post-open hooks at 1733500893634 (+4 ms)Region opened successfully at 1733500893634 2024-12-06T16:01:33,635 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., pid=57, masterSystemTime=1733500893617 2024-12-06T16:01:33,637 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:33,637 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:33,637 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=51b4277372e700eb3a1230f9ab87509e, regionState=OPEN, openSeqNum=2, regionLocation=85bef17d9292,46105,1733500827732 2024-12-06T16:01:33,639 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure 51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 because future has completed 2024-12-06T16:01:33,641 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-06T16:01:33,641 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; OpenRegionProcedure 51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 in 174 msec 2024-12-06T16:01:33,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-12-06T16:01:33,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=51b4277372e700eb3a1230f9ab87509e, ASSIGN in 331 msec 2024-12-06T16:01:33,643 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T16:01:33,643 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733500893643"}]},"ts":"1733500893643"} 2024-12-06T16:01:33,645 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-06T16:01:33,645 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T16:01:33,647 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1580 sec 2024-12-06T16:01:34,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-06T16:01:34,632 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:01:34,633 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ba7ffab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,633 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:01:34,633 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:01:34,634 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:01:34,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:01:34,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:01:34,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74d0812d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:01:34,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:01:34,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:34,637 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43434, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:01:34,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d2f9bfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,638 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:01:34,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:01:34,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:01:34,639 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59406, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:01:34,640 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T16:01:34,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 85bef17d9292,45907,1733500826847 2024-12-06T16:01:34,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@506d6437 2024-12-06T16:01:34,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T16:01:34,642 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43436, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T16:01:34,643 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11961dfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,643 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:01:34,643 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:01:34,644 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:01:34,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:01:34,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:01:34,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f093ef6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:01:34,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:01:34,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:34,645 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43454, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:01:34,646 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19c0e234, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,646 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48345386, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,647 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:01:34,647 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:01:34,647 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:01:34,647 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:01:34,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:01:34,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d6f1727, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:01:34,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:01:34,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:34,649 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43472, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:01:34,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b8a4074, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,650 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70ddee8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,650 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:01:34,650 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:01:34,651 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:01:34,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:01:34,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:01:34,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2500c2e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:01:34,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:01:34,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:34,652 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43482, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:01:34,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45fcdab4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,653 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5138abf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,653 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:01:34,654 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:01:34,654 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:01:34,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:01:34,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:01:34,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d8f4a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:01:34,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:01:34,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:34,656 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43508, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:01:34,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e89e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,657 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2108806c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,657 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:01:34,657 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:01:34,657 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:01:34,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:01:34,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:01:34,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f040a5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:01:34,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:01:34,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:34,659 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43532, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:01:34,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4be61725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,660 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2847dac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,660 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:01:34,660 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:01:34,661 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:01:34,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:01:34,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:01:34,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7946c73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:01:34,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:01:34,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:34,662 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43558, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:01:34,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@777ec361, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,663 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@289dbcde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,663 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:01:34,664 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:01:34,664 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:01:34,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:01:34,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:01:34,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7baf18e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:01:34,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:01:34,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:34,666 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43578, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:01:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@653db253, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@832215d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,668 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:01:34,668 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:01:34,669 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:01:34,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:01:34,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:01:34,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ef1b54a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:01:34,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:01:34,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:34,670 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43598, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:01:34,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e26b1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,671 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@478bceb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,671 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:01:34,671 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:01:34,672 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:01:34,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:01:34,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:01:34,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d83b292, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:01:34,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:01:34,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:34,673 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43620, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:01:34,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31dc62e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63eadf7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,674 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:01:34,674 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:01:34,675 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:01:34,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:01:34,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:01:34,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58d56c51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:01:34,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:01:34,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:34,676 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43648, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:01:34,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d0ece71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:01:34,677 DEBUG [Thread-1221 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:01:34,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:01:34,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:01:34,679 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:01:34,680 DEBUG [Thread-1223 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:01:34,680 DEBUG [Thread-1232 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:01:34,681 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:01:34,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:01:34,681 DEBUG [Thread-1223 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:01:34,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2] 2024-12-06T16:01:34,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:01:34,681 DEBUG [Thread-1223 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:01:34,681 DEBUG [Thread-1217 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:01:34,682 DEBUG [Thread-1219 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:01:34,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:01:34,682 DEBUG [Thread-1234 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:01:34,682 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59432, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:01:34,682 DEBUG [Thread-1225 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:01:34,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:01:34,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:01:34,683 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59426, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:01:34,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:01:34,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59442, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:01:34,684 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:01:34,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:01:34,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:01:34,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:01:34,685 DEBUG [Thread-1228 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:01:34,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2] 2024-12-06T16:01:34,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:01:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-12-06T16:01:34,686 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:01:34,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2] 2024-12-06T16:01:34,687 DEBUG [Thread-1228 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:01:34,687 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59468, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:01:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-06T16:01:34,688 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59460, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:01:34,689 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59482, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:01:34,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2] 2024-12-06T16:01:34,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2] 2024-12-06T16:01:34,691 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:01:34,691 DEBUG [Thread-1236 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:01:34,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2] 2024-12-06T16:01:34,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2] 2024-12-06T16:01:34,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:01:34,692 DEBUG [Thread-1223 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2] 2024-12-06T16:01:34,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:01:34,694 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59484, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:01:34,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2] 2024-12-06T16:01:34,696 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:01:34,696 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:01:34,699 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:01:34,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 51b4277372e700eb3a1230f9ab87509e 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-06T16:01:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:34,702 DEBUG [Thread-1230 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:01:34,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:01:34,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:01:34,704 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:01:34,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2] 2024-12-06T16:01:34,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500954738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:34,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:34,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:34,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:34,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:34,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500954740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:34,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:34,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500954741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:34,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:34,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:34,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:34,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:34,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:34,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:34,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500954748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:34,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:34,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:34,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:34,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:34,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500954749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:34,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:34,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:34,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:34,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/d4769b9638f7432c89cb877f718e5f00 is 50, key is test_row_0/A:col10/1733500894699/Put/seqid=0 2024-12-06T16:01:34,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742089_1265 (size=12001) 2024-12-06T16:01:34,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/d4769b9638f7432c89cb877f718e5f00 2024-12-06T16:01:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-06T16:01:34,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-06T16:01:34,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:34,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:34,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:34,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:34,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:34,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:34,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/27d16d397cb646809e6703427cd44209 is 50, key is test_row_0/B:col10/1733500894699/Put/seqid=0 2024-12-06T16:01:34,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:34,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500954852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:34,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:34,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:34,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:34,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742090_1266 (size=12001) 2024-12-06T16:01:34,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500954862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:34,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:34,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:34,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:34,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500954862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:34,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:34,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:34,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:34,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500954863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:34,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:34,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:34,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:34,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:34,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:34,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500954863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:34,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:34,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:34,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:35,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-06T16:01:35,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:35,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:35,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:35,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:35,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:35,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:35,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-06T16:01:35,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:35,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500955061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:35,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:35,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:35,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500955072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:35,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:35,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:35,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500955072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:35,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:35,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500955072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:35,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:35,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:35,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500955073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:35,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:35,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:35,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-06T16:01:35,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:35,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:35,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:35,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:35,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:35,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:35,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/27d16d397cb646809e6703427cd44209 2024-12-06T16:01:35,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/2e12751d30b746289abc3489e30fa292 is 50, key is test_row_0/C:col10/1733500894699/Put/seqid=0 2024-12-06T16:01:35,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-06T16:01:35,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:35,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:35,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:35,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:35,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:35,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742091_1267 (size=12001) 2024-12-06T16:01:35,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:35,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-06T16:01:35,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/2e12751d30b746289abc3489e30fa292 2024-12-06T16:01:35,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/d4769b9638f7432c89cb877f718e5f00 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/d4769b9638f7432c89cb877f718e5f00 2024-12-06T16:01:35,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/d4769b9638f7432c89cb877f718e5f00, entries=150, sequenceid=15, filesize=11.7 K 2024-12-06T16:01:35,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/27d16d397cb646809e6703427cd44209 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/27d16d397cb646809e6703427cd44209 2024-12-06T16:01:35,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/27d16d397cb646809e6703427cd44209, entries=150, sequenceid=15, filesize=11.7 K 2024-12-06T16:01:35,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/2e12751d30b746289abc3489e30fa292 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/2e12751d30b746289abc3489e30fa292 2024-12-06T16:01:35,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/2e12751d30b746289abc3489e30fa292, entries=150, sequenceid=15, filesize=11.7 K 2024-12-06T16:01:35,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 51b4277372e700eb3a1230f9ab87509e in 656ms, sequenceid=15, compaction requested=false 2024-12-06T16:01:35,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:35,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:35,375 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:01:35,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 51b4277372e700eb3a1230f9ab87509e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-06T16:01:35,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/04a4c35c74ad4df8a99c0ce8c23ac03b is 50, key is test_row_0/A:col10/1733500895374/Put/seqid=0 2024-12-06T16:01:35,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:35,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500955394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:35,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:35,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:35,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:35,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500955396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:35,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500955397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:35,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:35,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500955401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:35,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500955398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:35,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:35,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:35,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742092_1268 (size=14341) 2024-12-06T16:01:35,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/04a4c35c74ad4df8a99c0ce8c23ac03b 2024-12-06T16:01:35,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/c6abaf7986ce4655809425d17ce64e56 is 50, key is test_row_0/B:col10/1733500895374/Put/seqid=0 2024-12-06T16:01:35,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742093_1269 (size=12001) 2024-12-06T16:01:35,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-06T16:01:35,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:35,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:35,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:35,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:35,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:35,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:35,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:35,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500955501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:35,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:35,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:35,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500955512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:35,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:35,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:35,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500955512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:35,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:35,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:35,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:35,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500955512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:35,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500955512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:35,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:35,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:35,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-06T16:01:35,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:35,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:35,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:35,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:35,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:35,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:35,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:35,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500955711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:35,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500955722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:35,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:35,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500955722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:35,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500955722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:35,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500955722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:35,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-06T16:01:35,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:35,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:35,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:35,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:35,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-06T16:01:35,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/c6abaf7986ce4655809425d17ce64e56 2024-12-06T16:01:35,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/a59d9be6e0584c8ea32fc8325ac954fe is 50, key is test_row_0/C:col10/1733500895374/Put/seqid=0 2024-12-06T16:01:35,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742094_1270 (size=12001) 2024-12-06T16:01:35,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/a59d9be6e0584c8ea32fc8325ac954fe 2024-12-06T16:01:35,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/04a4c35c74ad4df8a99c0ce8c23ac03b as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/04a4c35c74ad4df8a99c0ce8c23ac03b 2024-12-06T16:01:35,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/04a4c35c74ad4df8a99c0ce8c23ac03b, entries=200, sequenceid=39, filesize=14.0 K 2024-12-06T16:01:35,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/c6abaf7986ce4655809425d17ce64e56 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/c6abaf7986ce4655809425d17ce64e56 2024-12-06T16:01:35,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/c6abaf7986ce4655809425d17ce64e56, entries=150, sequenceid=39, filesize=11.7 K 2024-12-06T16:01:35,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/a59d9be6e0584c8ea32fc8325ac954fe as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/a59d9be6e0584c8ea32fc8325ac954fe 2024-12-06T16:01:35,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/a59d9be6e0584c8ea32fc8325ac954fe, entries=150, sequenceid=39, filesize=11.7 K 2024-12-06T16:01:35,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 51b4277372e700eb3a1230f9ab87509e in 546ms, sequenceid=39, compaction requested=false 2024-12-06T16:01:35,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:35,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-06T16:01:35,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:35,931 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 51b4277372e700eb3a1230f9ab87509e 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-06T16:01:35,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/a76e1bc1c05d473db38f4463b07bc397 is 50, key is test_row_0/A:col10/1733500895399/Put/seqid=0 2024-12-06T16:01:35,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742095_1271 (size=12001) 2024-12-06T16:01:36,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:36,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:36,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:36,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500956058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:36,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:36,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500956060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:36,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:36,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:36,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:36,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500956061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:36,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:36,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500956064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:36,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:36,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:36,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:36,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500956067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:36,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:36,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:36,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500956172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:36,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:36,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500956172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:36,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:36,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:36,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500956172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:36,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:36,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:36,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:36,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500956182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:36,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:36,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500956182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:36,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:36,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:36,363 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/a76e1bc1c05d473db38f4463b07bc397 2024-12-06T16:01:36,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/147c6e72cea54d569c8598e8e7922fb9 is 50, key is test_row_0/B:col10/1733500895399/Put/seqid=0 2024-12-06T16:01:36,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:36,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500956381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:36,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:36,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:36,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500956383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:36,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:36,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:36,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500956382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:36,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:36,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:36,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500956392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:36,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:36,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500956392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:36,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:36,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:36,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742096_1272 (size=12001) 2024-12-06T16:01:36,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:36,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500956692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:36,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500956692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:36,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:36,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500956702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:36,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:36,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:36,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500956711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:36,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:36,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:36,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500956712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:36,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:36,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:36,815 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/147c6e72cea54d569c8598e8e7922fb9 2024-12-06T16:01:36,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/47f96edccaa6479898962a74670fb262 is 50, key is test_row_0/C:col10/1733500895399/Put/seqid=0 2024-12-06T16:01:36,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-06T16:01:36,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742097_1273 (size=12001) 2024-12-06T16:01:37,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:37,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500957205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:37,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:37,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:37,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:37,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:37,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500957205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:37,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:37,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:37,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:37,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:37,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500957212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:37,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:37,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:37,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:37,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500957222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:37,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:37,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:37,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:37,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500957222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:37,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:37,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:37,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:37,264 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/47f96edccaa6479898962a74670fb262 2024-12-06T16:01:37,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/a76e1bc1c05d473db38f4463b07bc397 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/a76e1bc1c05d473db38f4463b07bc397 2024-12-06T16:01:37,277 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/a76e1bc1c05d473db38f4463b07bc397, entries=150, sequenceid=52, filesize=11.7 K 2024-12-06T16:01:37,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/147c6e72cea54d569c8598e8e7922fb9 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/147c6e72cea54d569c8598e8e7922fb9 2024-12-06T16:01:37,279 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T16:01:37,290 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/147c6e72cea54d569c8598e8e7922fb9, entries=150, sequenceid=52, filesize=11.7 K 2024-12-06T16:01:37,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/47f96edccaa6479898962a74670fb262 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/47f96edccaa6479898962a74670fb262 2024-12-06T16:01:37,307 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/47f96edccaa6479898962a74670fb262, entries=150, sequenceid=52, filesize=11.7 K 2024-12-06T16:01:37,311 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 51b4277372e700eb3a1230f9ab87509e in 1380ms, sequenceid=52, compaction requested=true 2024-12-06T16:01:37,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:37,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:37,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-12-06T16:01:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-12-06T16:01:37,317 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-06T16:01:37,317 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6170 sec 2024-12-06T16:01:37,321 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 2.6370 sec 2024-12-06T16:01:38,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:38,217 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:01:38,217 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 51b4277372e700eb3a1230f9ab87509e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-06T16:01:38,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/c5166b912a0b42e48726d05d12bbed36 is 50, key is test_row_0/A:col10/1733500898215/Put/seqid=0 2024-12-06T16:01:38,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742098_1274 (size=14341) 2024-12-06T16:01:38,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:38,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500958237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:38,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:38,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:38,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500958238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:38,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:38,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:38,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500958240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:38,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:38,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:38,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500958242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:38,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:38,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:38,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500958246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:38,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:38,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:38,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:38,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500958352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:38,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500958353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:38,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:38,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:38,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500958353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:38,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:38,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500958353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:38,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:38,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:38,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:38,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:38,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500958361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:38,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:38,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500958570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:38,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:38,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500958570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:38,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:38,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500958572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:38,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:38,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:38,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500958572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:38,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:38,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500958573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:38,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:38,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:38,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:38,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/c5166b912a0b42e48726d05d12bbed36 2024-12-06T16:01:38,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/ed176bccfb494baba0490be5fe13fc0a is 50, key is test_row_0/B:col10/1733500898215/Put/seqid=0 2024-12-06T16:01:38,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742099_1275 (size=12001) 2024-12-06T16:01:38,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-06T16:01:38,862 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:01:38,863 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:01:38,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-12-06T16:01:38,866 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:01:38,867 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:01:38,867 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:01:38,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-06T16:01:38,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:38,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500958883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:38,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:38,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:38,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500958883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:38,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:38,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:38,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500958884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:38,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:38,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500958884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:38,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:38,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500958884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:38,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:38,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:38,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:38,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:38,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-06T16:01:39,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-06T16:01:39,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:39,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:39,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:39,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:39,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:39,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:39,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/ed176bccfb494baba0490be5fe13fc0a 2024-12-06T16:01:39,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/bc1f62efb40d4c4a9399f9cc16b207dc is 50, key is test_row_0/C:col10/1733500898215/Put/seqid=0 2024-12-06T16:01:39,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742100_1276 (size=12001) 2024-12-06T16:01:39,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-06T16:01:39,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:39,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:39,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:39,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:39,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:39,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-06T16:01:39,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:39,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-06T16:01:39,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:39,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:39,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:39,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:39,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:39,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:39,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:39,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500959392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:39,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:39,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:39,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:39,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:39,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500959392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:39,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:39,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:39,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:39,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:39,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500959402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:39,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:39,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500959402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:39,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:39,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:39,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:39,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:39,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500959402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:39,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:39,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:39,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:39,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-06T16:01:39,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:39,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:39,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:39,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:39,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:39,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:39,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-06T16:01:39,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/bc1f62efb40d4c4a9399f9cc16b207dc 2024-12-06T16:01:39,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/c5166b912a0b42e48726d05d12bbed36 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/c5166b912a0b42e48726d05d12bbed36 2024-12-06T16:01:39,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/c5166b912a0b42e48726d05d12bbed36, entries=200, sequenceid=77, filesize=14.0 K 2024-12-06T16:01:39,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/ed176bccfb494baba0490be5fe13fc0a as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/ed176bccfb494baba0490be5fe13fc0a 2024-12-06T16:01:39,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/ed176bccfb494baba0490be5fe13fc0a, entries=150, sequenceid=77, filesize=11.7 K 2024-12-06T16:01:39,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/bc1f62efb40d4c4a9399f9cc16b207dc as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/bc1f62efb40d4c4a9399f9cc16b207dc 2024-12-06T16:01:39,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/bc1f62efb40d4c4a9399f9cc16b207dc, entries=150, sequenceid=77, filesize=11.7 K 2024-12-06T16:01:39,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 51b4277372e700eb3a1230f9ab87509e in 1356ms, sequenceid=77, compaction requested=true 2024-12-06T16:01:39,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:39,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51b4277372e700eb3a1230f9ab87509e:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:01:39,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:39,573 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T16:01:39,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51b4277372e700eb3a1230f9ab87509e:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:01:39,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:39,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51b4277372e700eb3a1230f9ab87509e:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:01:39,574 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T16:01:39,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:39,575 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T16:01:39,575 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T16:01:39,575 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 51b4277372e700eb3a1230f9ab87509e/B is initiating minor compaction (all files) 2024-12-06T16:01:39,575 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 51b4277372e700eb3a1230f9ab87509e/A is initiating minor compaction (all files) 2024-12-06T16:01:39,575 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51b4277372e700eb3a1230f9ab87509e/A in TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:39,575 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51b4277372e700eb3a1230f9ab87509e/B in TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:39,575 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/d4769b9638f7432c89cb877f718e5f00, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/04a4c35c74ad4df8a99c0ce8c23ac03b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/a76e1bc1c05d473db38f4463b07bc397, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/c5166b912a0b42e48726d05d12bbed36] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp, totalSize=51.4 K 2024-12-06T16:01:39,575 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/27d16d397cb646809e6703427cd44209, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/c6abaf7986ce4655809425d17ce64e56, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/147c6e72cea54d569c8598e8e7922fb9, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/ed176bccfb494baba0490be5fe13fc0a] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp, totalSize=46.9 K 2024-12-06T16:01:39,576 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4769b9638f7432c89cb877f718e5f00, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733500894696 2024-12-06T16:01:39,576 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 27d16d397cb646809e6703427cd44209, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733500894696 2024-12-06T16:01:39,576 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04a4c35c74ad4df8a99c0ce8c23ac03b, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1733500894733 2024-12-06T16:01:39,576 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting a76e1bc1c05d473db38f4463b07bc397, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733500895394 2024-12-06T16:01:39,577 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting c6abaf7986ce4655809425d17ce64e56, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1733500894733 2024-12-06T16:01:39,578 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5166b912a0b42e48726d05d12bbed36, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733500896062 2024-12-06T16:01:39,578 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 147c6e72cea54d569c8598e8e7922fb9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733500895394 2024-12-06T16:01:39,579 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting ed176bccfb494baba0490be5fe13fc0a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733500896062 2024-12-06T16:01:39,593 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51b4277372e700eb3a1230f9ab87509e#A#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:39,593 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/e6bf054385c74a03894b1ddeefe6938f is 50, key is test_row_0/A:col10/1733500898215/Put/seqid=0 2024-12-06T16:01:39,602 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51b4277372e700eb3a1230f9ab87509e#B#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:39,603 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/7869b0138be74fafa661b787d5ec9239 is 50, key is test_row_0/B:col10/1733500898215/Put/seqid=0 2024-12-06T16:01:39,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742101_1277 (size=12139) 2024-12-06T16:01:39,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742102_1278 (size=12139) 2024-12-06T16:01:39,618 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/e6bf054385c74a03894b1ddeefe6938f as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/e6bf054385c74a03894b1ddeefe6938f 2024-12-06T16:01:39,630 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 51b4277372e700eb3a1230f9ab87509e/A of 51b4277372e700eb3a1230f9ab87509e into e6bf054385c74a03894b1ddeefe6938f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:39,630 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:39,630 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., storeName=51b4277372e700eb3a1230f9ab87509e/A, priority=12, startTime=1733500899573; duration=0sec 2024-12-06T16:01:39,630 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:39,630 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51b4277372e700eb3a1230f9ab87509e:A 2024-12-06T16:01:39,630 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T16:01:39,632 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T16:01:39,632 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 51b4277372e700eb3a1230f9ab87509e/C is initiating minor compaction (all files) 2024-12-06T16:01:39,632 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51b4277372e700eb3a1230f9ab87509e/C in TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:39,632 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/2e12751d30b746289abc3489e30fa292, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/a59d9be6e0584c8ea32fc8325ac954fe, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/47f96edccaa6479898962a74670fb262, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/bc1f62efb40d4c4a9399f9cc16b207dc] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp, totalSize=46.9 K 2024-12-06T16:01:39,633 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e12751d30b746289abc3489e30fa292, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733500894696 2024-12-06T16:01:39,633 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting a59d9be6e0584c8ea32fc8325ac954fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1733500894733 2024-12-06T16:01:39,633 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47f96edccaa6479898962a74670fb262, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733500895394 2024-12-06T16:01:39,634 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc1f62efb40d4c4a9399f9cc16b207dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733500896062 2024-12-06T16:01:39,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-06T16:01:39,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:39,644 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 51b4277372e700eb3a1230f9ab87509e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-06T16:01:39,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/446fe5bb56de445688f004e696d64274 is 50, key is test_row_0/A:col10/1733500898234/Put/seqid=0 2024-12-06T16:01:39,665 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51b4277372e700eb3a1230f9ab87509e#C#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:39,666 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/90a3c06c1fb94f56827fcdf2f03bbcf5 is 50, key is test_row_0/C:col10/1733500898215/Put/seqid=0 2024-12-06T16:01:39,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742103_1279 (size=12001) 2024-12-06T16:01:39,679 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/446fe5bb56de445688f004e696d64274 2024-12-06T16:01:39,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/766aea599d9d4534949235e9daa7f3ca is 50, key is test_row_0/B:col10/1733500898234/Put/seqid=0 2024-12-06T16:01:39,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742104_1280 (size=12139) 2024-12-06T16:01:39,714 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/90a3c06c1fb94f56827fcdf2f03bbcf5 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/90a3c06c1fb94f56827fcdf2f03bbcf5 2024-12-06T16:01:39,721 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 51b4277372e700eb3a1230f9ab87509e/C of 51b4277372e700eb3a1230f9ab87509e into 90a3c06c1fb94f56827fcdf2f03bbcf5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:39,721 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:39,721 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., storeName=51b4277372e700eb3a1230f9ab87509e/C, priority=12, startTime=1733500899574; duration=0sec 2024-12-06T16:01:39,721 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:39,721 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51b4277372e700eb3a1230f9ab87509e:C 2024-12-06T16:01:39,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742105_1281 (size=12001) 2024-12-06T16:01:40,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-06T16:01:40,036 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/7869b0138be74fafa661b787d5ec9239 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/7869b0138be74fafa661b787d5ec9239 2024-12-06T16:01:40,043 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 51b4277372e700eb3a1230f9ab87509e/B of 51b4277372e700eb3a1230f9ab87509e into 7869b0138be74fafa661b787d5ec9239(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:40,043 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:40,043 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., storeName=51b4277372e700eb3a1230f9ab87509e/B, priority=12, startTime=1733500899573; duration=0sec 2024-12-06T16:01:40,043 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:40,043 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51b4277372e700eb3a1230f9ab87509e:B 2024-12-06T16:01:40,139 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/766aea599d9d4534949235e9daa7f3ca 2024-12-06T16:01:40,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/5c2f7341034f4ec8bc28c19d090bb37c is 50, key is test_row_0/C:col10/1733500898234/Put/seqid=0 2024-12-06T16:01:40,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742106_1282 (size=12001) 2024-12-06T16:01:40,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:40,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:40,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:40,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500960441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:40,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:40,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:40,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:40,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:40,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500960443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:40,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:40,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:40,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:40,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:40,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500960444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:40,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:40,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:40,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:40,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:40,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500960446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:40,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:40,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:40,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:40,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:40,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500960446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:40,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:40,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:40,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:40,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:40,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500960552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:40,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:40,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:40,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:40,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:40,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500960552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:40,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:40,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:40,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:40,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:40,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500960552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:40,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:40,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:40,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:40,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:40,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500960562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:40,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:40,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:40,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:40,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:40,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500960562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:40,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:40,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:40,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:40,580 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/5c2f7341034f4ec8bc28c19d090bb37c 2024-12-06T16:01:40,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/446fe5bb56de445688f004e696d64274 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/446fe5bb56de445688f004e696d64274 2024-12-06T16:01:40,594 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/446fe5bb56de445688f004e696d64274, entries=150, sequenceid=89, filesize=11.7 K 2024-12-06T16:01:40,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/766aea599d9d4534949235e9daa7f3ca as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/766aea599d9d4534949235e9daa7f3ca 2024-12-06T16:01:40,601 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/766aea599d9d4534949235e9daa7f3ca, entries=150, sequenceid=89, filesize=11.7 K 2024-12-06T16:01:40,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/5c2f7341034f4ec8bc28c19d090bb37c as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/5c2f7341034f4ec8bc28c19d090bb37c 2024-12-06T16:01:40,609 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/5c2f7341034f4ec8bc28c19d090bb37c, entries=150, sequenceid=89, filesize=11.7 K 2024-12-06T16:01:40,610 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 51b4277372e700eb3a1230f9ab87509e in 967ms, sequenceid=89, compaction requested=false 2024-12-06T16:01:40,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:40,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:40,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-12-06T16:01:40,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-12-06T16:01:40,614 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-06T16:01:40,614 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7450 sec 2024-12-06T16:01:40,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.7520 sec 2024-12-06T16:01:40,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:40,764 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:01:40,765 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 51b4277372e700eb3a1230f9ab87509e 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-12-06T16:01:40,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/0ab02b9940a24ded910174bddb18018f is 50, key is test_row_0/A:col10/1733500900765/Put/seqid=0 2024-12-06T16:01:40,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:40,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500960772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:40,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:40,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:40,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:40,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:40,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500960772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:40,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:40,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500960772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:40,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:40,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500960773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:40,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:40,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500960775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:40,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742107_1283 (size=14341) 2024-12-06T16:01:40,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:40,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:40,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:40,780 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/0ab02b9940a24ded910174bddb18018f 2024-12-06T16:01:40,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/3e3096822aab4f3aaf10f5b117673812 is 50, key is test_row_0/B:col10/1733500900765/Put/seqid=0 2024-12-06T16:01:40,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742108_1284 (size=12001) 2024-12-06T16:01:40,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/3e3096822aab4f3aaf10f5b117673812 2024-12-06T16:01:40,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/817ce68722fd4d5cb2ea19906156bc52 is 50, key is test_row_0/C:col10/1733500900765/Put/seqid=0 2024-12-06T16:01:40,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742109_1285 (size=12001) 2024-12-06T16:01:40,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/817ce68722fd4d5cb2ea19906156bc52 2024-12-06T16:01:40,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/0ab02b9940a24ded910174bddb18018f as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/0ab02b9940a24ded910174bddb18018f 2024-12-06T16:01:40,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/0ab02b9940a24ded910174bddb18018f, entries=200, sequenceid=119, filesize=14.0 K 2024-12-06T16:01:40,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/3e3096822aab4f3aaf10f5b117673812 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/3e3096822aab4f3aaf10f5b117673812 2024-12-06T16:01:40,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/3e3096822aab4f3aaf10f5b117673812, entries=150, sequenceid=119, filesize=11.7 K 2024-12-06T16:01:40,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/817ce68722fd4d5cb2ea19906156bc52 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/817ce68722fd4d5cb2ea19906156bc52 2024-12-06T16:01:40,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/817ce68722fd4d5cb2ea19906156bc52, entries=150, sequenceid=119, filesize=11.7 K 2024-12-06T16:01:40,843 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 51b4277372e700eb3a1230f9ab87509e in 79ms, sequenceid=119, compaction requested=true 2024-12-06T16:01:40,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:40,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51b4277372e700eb3a1230f9ab87509e:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:01:40,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:40,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51b4277372e700eb3a1230f9ab87509e:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:01:40,843 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:40,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:40,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51b4277372e700eb3a1230f9ab87509e:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:01:40,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-06T16:01:40,843 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:40,848 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:40,848 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 51b4277372e700eb3a1230f9ab87509e/A is initiating minor compaction (all files) 2024-12-06T16:01:40,848 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51b4277372e700eb3a1230f9ab87509e/A in TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:40,848 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/e6bf054385c74a03894b1ddeefe6938f, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/446fe5bb56de445688f004e696d64274, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/0ab02b9940a24ded910174bddb18018f] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp, totalSize=37.6 K 2024-12-06T16:01:40,849 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:40,849 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6bf054385c74a03894b1ddeefe6938f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733500896062 2024-12-06T16:01:40,849 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 51b4277372e700eb3a1230f9ab87509e/B is initiating minor compaction (all files) 2024-12-06T16:01:40,849 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51b4277372e700eb3a1230f9ab87509e/B in TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:40,849 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/7869b0138be74fafa661b787d5ec9239, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/766aea599d9d4534949235e9daa7f3ca, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/3e3096822aab4f3aaf10f5b117673812] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp, totalSize=35.3 K 2024-12-06T16:01:40,851 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 7869b0138be74fafa661b787d5ec9239, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733500896062 2024-12-06T16:01:40,851 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 446fe5bb56de445688f004e696d64274, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733500898234 2024-12-06T16:01:40,851 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 766aea599d9d4534949235e9daa7f3ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733500898234 2024-12-06T16:01:40,851 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ab02b9940a24ded910174bddb18018f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733500900440 2024-12-06T16:01:40,852 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e3096822aab4f3aaf10f5b117673812, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733500900440 2024-12-06T16:01:40,862 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51b4277372e700eb3a1230f9ab87509e#A#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:40,862 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/02dbe71a7c7e4362a09db13b7153652b is 50, key is test_row_0/A:col10/1733500900765/Put/seqid=0 2024-12-06T16:01:40,886 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51b4277372e700eb3a1230f9ab87509e#B#compaction#238 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:40,887 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/5f12385f3e464cf4aafdf98ffe60c61e is 50, key is test_row_0/B:col10/1733500900765/Put/seqid=0 2024-12-06T16:01:40,894 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:01:40,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 51b4277372e700eb3a1230f9ab87509e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-06T16:01:40,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:40,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742110_1286 (size=12241) 2024-12-06T16:01:40,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742111_1287 (size=12241) 2024-12-06T16:01:40,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/f3e2f540b5da494dbd3342a1d7ecafaa is 50, key is test_row_0/A:col10/1733500900886/Put/seqid=0 2024-12-06T16:01:40,913 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/5f12385f3e464cf4aafdf98ffe60c61e as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/5f12385f3e464cf4aafdf98ffe60c61e 2024-12-06T16:01:40,929 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51b4277372e700eb3a1230f9ab87509e/B of 51b4277372e700eb3a1230f9ab87509e into 5f12385f3e464cf4aafdf98ffe60c61e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:40,929 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:40,929 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., storeName=51b4277372e700eb3a1230f9ab87509e/B, priority=13, startTime=1733500900843; duration=0sec 2024-12-06T16:01:40,929 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:40,929 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51b4277372e700eb3a1230f9ab87509e:B 2024-12-06T16:01:40,929 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:40,932 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:40,933 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 51b4277372e700eb3a1230f9ab87509e/C is initiating minor compaction (all files) 2024-12-06T16:01:40,933 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51b4277372e700eb3a1230f9ab87509e/C in TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:40,933 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/90a3c06c1fb94f56827fcdf2f03bbcf5, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/5c2f7341034f4ec8bc28c19d090bb37c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/817ce68722fd4d5cb2ea19906156bc52] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp, totalSize=35.3 K 2024-12-06T16:01:40,933 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 90a3c06c1fb94f56827fcdf2f03bbcf5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733500896062 2024-12-06T16:01:40,934 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c2f7341034f4ec8bc28c19d090bb37c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733500898234 2024-12-06T16:01:40,934 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 817ce68722fd4d5cb2ea19906156bc52, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733500900440 2024-12-06T16:01:40,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742112_1288 (size=14441) 2024-12-06T16:01:40,957 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51b4277372e700eb3a1230f9ab87509e#C#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:40,958 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/710ceb21e8844739a161a19f2f20fe24 is 50, key is test_row_0/C:col10/1733500900765/Put/seqid=0 2024-12-06T16:01:40,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742113_1289 (size=12241) 2024-12-06T16:01:41,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:41,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500961005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:41,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-06T16:01:41,012 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:01:41,012 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:01:41,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-12-06T16:01:41,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:41,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500961009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:41,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-06T16:01:41,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:41,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500961010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:41,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:41,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:41,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:41,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:41,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:41,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:41,015 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:01:41,016 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:01:41,016 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:01:41,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:41,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500961082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:41,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:41,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:41,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:41,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500961082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:41,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:41,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:41,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:41,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:41,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-06T16:01:41,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:41,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500961122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:41,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:41,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:41,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:41,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:41,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500961123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:41,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:41,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:41,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:41,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:41,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500961123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:41,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:41,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:41,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:41,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-06T16:01:41,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:41,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:41,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:41,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:41,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:41,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:41,301 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/02dbe71a7c7e4362a09db13b7153652b as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/02dbe71a7c7e4362a09db13b7153652b 2024-12-06T16:01:41,308 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51b4277372e700eb3a1230f9ab87509e/A of 51b4277372e700eb3a1230f9ab87509e into 02dbe71a7c7e4362a09db13b7153652b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:41,308 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:41,308 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., storeName=51b4277372e700eb3a1230f9ab87509e/A, priority=13, startTime=1733500900843; duration=0sec 2024-12-06T16:01:41,308 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:41,308 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51b4277372e700eb3a1230f9ab87509e:A 2024-12-06T16:01:41,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-06T16:01:41,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:41,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:41,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:41,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:41,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:41,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-06T16:01:41,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:41,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:41,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500961332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:41,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500961332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:41,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:41,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500961332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:41,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:41,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:41,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:41,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:41,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:41,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:41,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:41,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:41,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:41,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/f3e2f540b5da494dbd3342a1d7ecafaa 2024-12-06T16:01:41,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/ad25b6143a9b4b9ea5fd7ba3952ebf1c is 50, key is test_row_0/B:col10/1733500900886/Put/seqid=0 2024-12-06T16:01:41,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742114_1290 (size=12101) 2024-12-06T16:01:41,384 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/710ceb21e8844739a161a19f2f20fe24 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/710ceb21e8844739a161a19f2f20fe24 2024-12-06T16:01:41,392 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51b4277372e700eb3a1230f9ab87509e/C of 51b4277372e700eb3a1230f9ab87509e into 710ceb21e8844739a161a19f2f20fe24(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:41,392 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:41,392 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., storeName=51b4277372e700eb3a1230f9ab87509e/C, priority=13, startTime=1733500900843; duration=0sec 2024-12-06T16:01:41,392 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:41,392 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51b4277372e700eb3a1230f9ab87509e:C 2024-12-06T16:01:41,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-06T16:01:41,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:41,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:41,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:41,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:41,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:41,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:41,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:41,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500961592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:41,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:41,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500961592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:41,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-06T16:01:41,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:41,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:41,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:41,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:41,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:41,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:41,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-06T16:01:41,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:41,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500961642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:41,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:41,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:41,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500961642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:41,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500961642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:41,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:41,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:41,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:41,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:41,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:41,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:41,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/ad25b6143a9b4b9ea5fd7ba3952ebf1c 2024-12-06T16:01:41,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/e797936106bb40a09e17849d69b0850d is 50, key is test_row_0/C:col10/1733500900886/Put/seqid=0 2024-12-06T16:01:41,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742115_1291 (size=12101) 2024-12-06T16:01:41,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/e797936106bb40a09e17849d69b0850d 2024-12-06T16:01:41,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-06T16:01:41,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:41,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:41,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:41,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:41,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:41,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:41,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/f3e2f540b5da494dbd3342a1d7ecafaa as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/f3e2f540b5da494dbd3342a1d7ecafaa 2024-12-06T16:01:41,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/f3e2f540b5da494dbd3342a1d7ecafaa, entries=200, sequenceid=130, filesize=14.1 K 2024-12-06T16:01:41,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/ad25b6143a9b4b9ea5fd7ba3952ebf1c as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/ad25b6143a9b4b9ea5fd7ba3952ebf1c 2024-12-06T16:01:41,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/ad25b6143a9b4b9ea5fd7ba3952ebf1c, entries=150, sequenceid=130, filesize=11.8 K 2024-12-06T16:01:41,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/e797936106bb40a09e17849d69b0850d as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/e797936106bb40a09e17849d69b0850d 2024-12-06T16:01:41,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/e797936106bb40a09e17849d69b0850d, entries=150, sequenceid=130, filesize=11.8 K 2024-12-06T16:01:41,806 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 51b4277372e700eb3a1230f9ab87509e in 912ms, sequenceid=130, compaction requested=false 2024-12-06T16:01:41,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:41,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-06T16:01:41,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:41,937 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 51b4277372e700eb3a1230f9ab87509e 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-06T16:01:41,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/e03471c527024cb3bc33c118bd25e63c is 50, key is test_row_0/A:col10/1733500900950/Put/seqid=0 2024-12-06T16:01:41,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742116_1292 (size=12151) 2024-12-06T16:01:42,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-06T16:01:42,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:42,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:42,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:42,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500962161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:42,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:42,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:42,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:42,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:42,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500962163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:42,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500962164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:42,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:42,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:42,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500962272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:42,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500962272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:42,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:42,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:42,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500962273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:42,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:42,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:42,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:42,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:42,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:42,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:42,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:42,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:42,372 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/e03471c527024cb3bc33c118bd25e63c 2024-12-06T16:01:42,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/4ecf9ab0dcf44e278dc91fe548583931 is 50, key is test_row_0/B:col10/1733500900950/Put/seqid=0 2024-12-06T16:01:42,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742117_1293 (size=12151) 2024-12-06T16:01:42,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:42,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500962482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:42,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:42,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500962482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:42,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:42,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:42,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:42,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:42,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500962482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:42,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:42,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:42,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:42,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:42,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:42,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:42,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:42,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500962602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:42,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:42,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500962602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:42,605 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 85bef17d9292,46105,1733500827732 for 'test_row_1' in 51b4277372e700eb3a1230f9ab87509e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T16:01:42,606 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 85bef17d9292,46105,1733500827732 for 'test_row_0' in 51b4277372e700eb3a1230f9ab87509e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T16:01:42,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:42,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:42,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:42,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:42,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:42,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:42,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:42,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500962792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:42,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:42,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:42,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:42,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:42,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500962792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:42,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:42,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500962792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:42,797 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/4ecf9ab0dcf44e278dc91fe548583931 2024-12-06T16:01:42,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/851c9d5a5b704de8aaf3c3445c28e3d1 is 50, key is test_row_0/C:col10/1733500900950/Put/seqid=0 2024-12-06T16:01:42,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742118_1294 (size=12151) 2024-12-06T16:01:42,833 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/851c9d5a5b704de8aaf3c3445c28e3d1 2024-12-06T16:01:42,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/e03471c527024cb3bc33c118bd25e63c as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/e03471c527024cb3bc33c118bd25e63c 2024-12-06T16:01:42,851 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/e03471c527024cb3bc33c118bd25e63c, entries=150, sequenceid=159, filesize=11.9 K 2024-12-06T16:01:42,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/4ecf9ab0dcf44e278dc91fe548583931 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/4ecf9ab0dcf44e278dc91fe548583931 2024-12-06T16:01:42,860 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/4ecf9ab0dcf44e278dc91fe548583931, entries=150, sequenceid=159, filesize=11.9 K 2024-12-06T16:01:42,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/851c9d5a5b704de8aaf3c3445c28e3d1 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/851c9d5a5b704de8aaf3c3445c28e3d1 2024-12-06T16:01:42,868 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/851c9d5a5b704de8aaf3c3445c28e3d1, entries=150, sequenceid=159, filesize=11.9 K 2024-12-06T16:01:42,869 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 51b4277372e700eb3a1230f9ab87509e in 932ms, sequenceid=159, compaction requested=true 2024-12-06T16:01:42,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:42,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:42,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-06T16:01:42,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-06T16:01:42,875 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-06T16:01:42,875 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8550 sec 2024-12-06T16:01:42,877 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.8630 sec 2024-12-06T16:01:43,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-06T16:01:43,163 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:01:43,163 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:01:43,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-06T16:01:43,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-06T16:01:43,166 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:01:43,167 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:01:43,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:01:43,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-06T16:01:43,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:43,305 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:01:43,306 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 51b4277372e700eb3a1230f9ab87509e 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-06T16:01:43,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/b5c2fc7bb9154bfeafa60aac0d62f0b5 is 50, key is test_row_0/A:col10/1733500903304/Put/seqid=0 2024-12-06T16:01:43,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-06T16:01:43,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:43,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:43,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:43,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:43,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:43,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:43,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742119_1295 (size=12151) 2024-12-06T16:01:43,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:43,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500963350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:43,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:43,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:43,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:43,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500963350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:43,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:43,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:43,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:43,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:43,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:43,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500963352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:43,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:43,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:43,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:43,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:43,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500963461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:43,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:43,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:43,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:43,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:43,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:43,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500963461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:43,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500963462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:43,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:43,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:43,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:43,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:43,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:43,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:43,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-06T16:01:43,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:43,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:43,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:43,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:43,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:43,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:43,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-06T16:01:43,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-06T16:01:43,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:43,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:43,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:43,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:43,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:43,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:43,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:43,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500963672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:43,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:43,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:43,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500963672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:43,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:43,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:43,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:43,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:43,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:43,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500963672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:43,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:43,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:43,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:43,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/b5c2fc7bb9154bfeafa60aac0d62f0b5 2024-12-06T16:01:43,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/51cf2f94795d4233b6a79aac8620d2d2 is 50, key is test_row_0/B:col10/1733500903304/Put/seqid=0 2024-12-06T16:01:43,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742120_1296 (size=12151) 2024-12-06T16:01:43,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-06T16:01:43,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:43,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:43,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:43,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:43,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-06T16:01:43,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-06T16:01:43,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:43,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:43,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:43,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:43,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:43,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:43,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500963982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:43,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:43,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500963982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:43,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:43,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500963982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:44,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-06T16:01:44,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:44,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:44,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:44,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:44,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:44,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/51cf2f94795d4233b6a79aac8620d2d2 2024-12-06T16:01:44,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/37d26045da874c629ae099ddf2c856b1 is 50, key is test_row_0/C:col10/1733500903304/Put/seqid=0 2024-12-06T16:01:44,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742121_1297 (size=12151) 2024-12-06T16:01:44,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-06T16:01:44,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:44,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:44,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:44,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:44,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:44,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:44,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-06T16:01:44,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-06T16:01:44,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:44,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:44,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:44,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:44,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:44,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:44,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500964492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:44,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:44,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:44,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500964492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:44,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:44,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:44,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:44,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:44,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:44,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:44,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500964501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:44,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:44,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:44,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:44,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-06T16:01:44,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:44,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:44,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:44,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:44,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:44,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:44,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/37d26045da874c629ae099ddf2c856b1 2024-12-06T16:01:44,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/b5c2fc7bb9154bfeafa60aac0d62f0b5 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/b5c2fc7bb9154bfeafa60aac0d62f0b5 2024-12-06T16:01:44,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/b5c2fc7bb9154bfeafa60aac0d62f0b5, entries=150, sequenceid=172, filesize=11.9 K 2024-12-06T16:01:44,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/51cf2f94795d4233b6a79aac8620d2d2 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/51cf2f94795d4233b6a79aac8620d2d2 2024-12-06T16:01:44,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/51cf2f94795d4233b6a79aac8620d2d2, entries=150, sequenceid=172, filesize=11.9 K 2024-12-06T16:01:44,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/37d26045da874c629ae099ddf2c856b1 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/37d26045da874c629ae099ddf2c856b1 2024-12-06T16:01:44,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/37d26045da874c629ae099ddf2c856b1, entries=150, sequenceid=172, filesize=11.9 K 2024-12-06T16:01:44,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 51b4277372e700eb3a1230f9ab87509e in 1305ms, sequenceid=172, compaction requested=true 2024-12-06T16:01:44,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:44,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51b4277372e700eb3a1230f9ab87509e:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:01:44,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:44,610 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T16:01:44,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51b4277372e700eb3a1230f9ab87509e:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:01:44,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:44,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51b4277372e700eb3a1230f9ab87509e:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:01:44,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:44,610 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T16:01:44,612 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50984 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T16:01:44,612 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 51b4277372e700eb3a1230f9ab87509e/A is initiating minor compaction (all files) 2024-12-06T16:01:44,612 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51b4277372e700eb3a1230f9ab87509e/A in TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:44,612 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/02dbe71a7c7e4362a09db13b7153652b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/f3e2f540b5da494dbd3342a1d7ecafaa, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/e03471c527024cb3bc33c118bd25e63c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/b5c2fc7bb9154bfeafa60aac0d62f0b5] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp, totalSize=49.8 K 2024-12-06T16:01:44,612 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T16:01:44,613 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 51b4277372e700eb3a1230f9ab87509e/B is initiating minor compaction (all files) 2024-12-06T16:01:44,613 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51b4277372e700eb3a1230f9ab87509e/B in TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:44,613 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02dbe71a7c7e4362a09db13b7153652b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733500900440 2024-12-06T16:01:44,613 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/5f12385f3e464cf4aafdf98ffe60c61e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/ad25b6143a9b4b9ea5fd7ba3952ebf1c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/4ecf9ab0dcf44e278dc91fe548583931, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/51cf2f94795d4233b6a79aac8620d2d2] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp, totalSize=47.5 K 2024-12-06T16:01:44,613 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3e2f540b5da494dbd3342a1d7ecafaa, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733500900770 2024-12-06T16:01:44,613 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f12385f3e464cf4aafdf98ffe60c61e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733500900440 2024-12-06T16:01:44,613 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting e03471c527024cb3bc33c118bd25e63c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733500900949 2024-12-06T16:01:44,614 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting ad25b6143a9b4b9ea5fd7ba3952ebf1c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733500900886 2024-12-06T16:01:44,614 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5c2fc7bb9154bfeafa60aac0d62f0b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733500902162 2024-12-06T16:01:44,614 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ecf9ab0dcf44e278dc91fe548583931, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733500900949 2024-12-06T16:01:44,614 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 51cf2f94795d4233b6a79aac8620d2d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733500902162 2024-12-06T16:01:44,625 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51b4277372e700eb3a1230f9ab87509e#A#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:44,626 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/6c87bebfa4e941cab80c0df69b629ea1 is 50, key is test_row_0/A:col10/1733500903304/Put/seqid=0 2024-12-06T16:01:44,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:44,627 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:01:44,627 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 51b4277372e700eb3a1230f9ab87509e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-06T16:01:44,642 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51b4277372e700eb3a1230f9ab87509e#B#compaction#250 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:44,643 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/4e174896b8d240c9bb9155141ee15fb9 is 50, key is test_row_0/B:col10/1733500903304/Put/seqid=0 2024-12-06T16:01:44,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/7608eadefda84735afbc320627b633da is 50, key is test_row_0/A:col10/1733500903350/Put/seqid=0 2024-12-06T16:01:44,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742123_1299 (size=12527) 2024-12-06T16:01:44,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742122_1298 (size=12527) 2024-12-06T16:01:44,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:44,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500964660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:44,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:44,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:44,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:44,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:44,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500964662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:44,672 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/4e174896b8d240c9bb9155141ee15fb9 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/4e174896b8d240c9bb9155141ee15fb9 2024-12-06T16:01:44,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742124_1300 (size=14541) 2024-12-06T16:01:44,688 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 51b4277372e700eb3a1230f9ab87509e/B of 51b4277372e700eb3a1230f9ab87509e into 4e174896b8d240c9bb9155141ee15fb9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:44,688 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:44,688 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., storeName=51b4277372e700eb3a1230f9ab87509e/B, priority=12, startTime=1733500904610; duration=0sec 2024-12-06T16:01:44,688 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:44,688 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51b4277372e700eb3a1230f9ab87509e:B 2024-12-06T16:01:44,688 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T16:01:44,691 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T16:01:44,691 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 51b4277372e700eb3a1230f9ab87509e/C is initiating minor compaction (all files) 2024-12-06T16:01:44,691 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51b4277372e700eb3a1230f9ab87509e/C in TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:44,691 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/710ceb21e8844739a161a19f2f20fe24, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/e797936106bb40a09e17849d69b0850d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/851c9d5a5b704de8aaf3c3445c28e3d1, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/37d26045da874c629ae099ddf2c856b1] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp, totalSize=47.5 K 2024-12-06T16:01:44,692 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 710ceb21e8844739a161a19f2f20fe24, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733500900440 2024-12-06T16:01:44,693 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting e797936106bb40a09e17849d69b0850d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733500900886 2024-12-06T16:01:44,694 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 851c9d5a5b704de8aaf3c3445c28e3d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733500900949 2024-12-06T16:01:44,694 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 37d26045da874c629ae099ddf2c856b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733500902162 2024-12-06T16:01:44,704 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51b4277372e700eb3a1230f9ab87509e#C#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:44,704 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/985fce4699bd412a88572535386c7c09 is 50, key is test_row_0/C:col10/1733500903304/Put/seqid=0 2024-12-06T16:01:44,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-06T16:01:44,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:44,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:44,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:44,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:44,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:44,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:44,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742125_1301 (size=12527) 2024-12-06T16:01:44,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:44,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:44,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500964772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:44,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500964771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:44,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:44,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:44,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:44,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:44,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:44,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:44,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-06T16:01:44,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:44,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:44,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:44,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:44,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:44,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:44,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:44,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500964981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:44,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:44,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:44,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500964981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:44,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:44,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:44,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:44,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:44,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:45,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-06T16:01:45,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:45,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:45,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:45,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:45,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:45,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:45,077 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/6c87bebfa4e941cab80c0df69b629ea1 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/6c87bebfa4e941cab80c0df69b629ea1 2024-12-06T16:01:45,083 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 51b4277372e700eb3a1230f9ab87509e/A of 51b4277372e700eb3a1230f9ab87509e into 6c87bebfa4e941cab80c0df69b629ea1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:45,083 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:45,083 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., storeName=51b4277372e700eb3a1230f9ab87509e/A, priority=12, startTime=1733500904610; duration=0sec 2024-12-06T16:01:45,083 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:45,083 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51b4277372e700eb3a1230f9ab87509e:A 2024-12-06T16:01:45,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/7608eadefda84735afbc320627b633da 2024-12-06T16:01:45,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/9cff1684d47a410f93ae4a3fe4b571a0 is 50, key is test_row_0/B:col10/1733500903350/Put/seqid=0 2024-12-06T16:01:45,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742126_1302 (size=12151) 2024-12-06T16:01:45,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/9cff1684d47a410f93ae4a3fe4b571a0 2024-12-06T16:01:45,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/dd56a78d708643e6892a679ecd3d1aeb is 50, key is test_row_0/C:col10/1733500903350/Put/seqid=0 2024-12-06T16:01:45,121 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/985fce4699bd412a88572535386c7c09 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/985fce4699bd412a88572535386c7c09 2024-12-06T16:01:45,126 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 51b4277372e700eb3a1230f9ab87509e/C of 51b4277372e700eb3a1230f9ab87509e into 985fce4699bd412a88572535386c7c09(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:45,126 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:45,126 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., storeName=51b4277372e700eb3a1230f9ab87509e/C, priority=12, startTime=1733500904610; duration=0sec 2024-12-06T16:01:45,126 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:45,126 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51b4277372e700eb3a1230f9ab87509e:C 2024-12-06T16:01:45,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742127_1303 (size=12151) 2024-12-06T16:01:45,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-06T16:01:45,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:45,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:45,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:45,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:45,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:45,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:45,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:45,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500965292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:45,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:45,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:45,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:45,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500965292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:45,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:45,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:45,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:45,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:45,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-06T16:01:45,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-06T16:01:45,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:45,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:45,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:45,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:45,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:45,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:45,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-06T16:01:45,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:45,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:45,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:45,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:45,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:45,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:45,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500965503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:45,507 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 85bef17d9292,46105,1733500827732 for 'test_row_0' in 51b4277372e700eb3a1230f9ab87509e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T16:01:45,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:45,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:45,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:45,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500965512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:45,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:45,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500965512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:45,515 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 85bef17d9292,46105,1733500827732 for 'test_row_1' in 51b4277372e700eb3a1230f9ab87509e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T16:01:45,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:45,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:45,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:45,516 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 85bef17d9292,46105,1733500827732 for 'test_row_1' in 51b4277372e700eb3a1230f9ab87509e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T16:01:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:45,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/dd56a78d708643e6892a679ecd3d1aeb 2024-12-06T16:01:45,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/7608eadefda84735afbc320627b633da as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/7608eadefda84735afbc320627b633da 2024-12-06T16:01:45,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/7608eadefda84735afbc320627b633da, entries=200, sequenceid=196, filesize=14.2 K 2024-12-06T16:01:45,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/9cff1684d47a410f93ae4a3fe4b571a0 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/9cff1684d47a410f93ae4a3fe4b571a0 2024-12-06T16:01:45,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/9cff1684d47a410f93ae4a3fe4b571a0, entries=150, sequenceid=196, filesize=11.9 K 2024-12-06T16:01:45,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/dd56a78d708643e6892a679ecd3d1aeb as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/dd56a78d708643e6892a679ecd3d1aeb 2024-12-06T16:01:45,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/dd56a78d708643e6892a679ecd3d1aeb, entries=150, sequenceid=196, filesize=11.9 K 2024-12-06T16:01:45,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 51b4277372e700eb3a1230f9ab87509e in 953ms, sequenceid=196, compaction requested=false 2024-12-06T16:01:45,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:45,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-06T16:01:45,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:45,629 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 51b4277372e700eb3a1230f9ab87509e 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-06T16:01:45,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/5655ea36bbaa4c5fac501b1da6ea79fc is 50, key is test_row_0/A:col10/1733500904659/Put/seqid=0 2024-12-06T16:01:45,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742128_1304 (size=9757) 2024-12-06T16:01:45,663 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/5655ea36bbaa4c5fac501b1da6ea79fc 2024-12-06T16:01:45,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/5214392df5b74a87a6a35745ae92f1bf is 50, key is test_row_0/B:col10/1733500904659/Put/seqid=0 2024-12-06T16:01:45,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742129_1305 (size=9757) 2024-12-06T16:01:45,693 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/5214392df5b74a87a6a35745ae92f1bf 2024-12-06T16:01:45,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/5e34ca72e1b1497b88bbb28cf0aa9e21 is 50, key is test_row_0/C:col10/1733500904659/Put/seqid=0 2024-12-06T16:01:45,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742130_1306 (size=9757) 2024-12-06T16:01:45,734 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/5e34ca72e1b1497b88bbb28cf0aa9e21 2024-12-06T16:01:45,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/5655ea36bbaa4c5fac501b1da6ea79fc as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/5655ea36bbaa4c5fac501b1da6ea79fc 2024-12-06T16:01:45,762 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/5655ea36bbaa4c5fac501b1da6ea79fc, entries=100, sequenceid=211, filesize=9.5 K 2024-12-06T16:01:45,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/5214392df5b74a87a6a35745ae92f1bf as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/5214392df5b74a87a6a35745ae92f1bf 2024-12-06T16:01:45,768 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/5214392df5b74a87a6a35745ae92f1bf, entries=100, sequenceid=211, filesize=9.5 K 2024-12-06T16:01:45,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/5e34ca72e1b1497b88bbb28cf0aa9e21 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/5e34ca72e1b1497b88bbb28cf0aa9e21 2024-12-06T16:01:45,778 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/5e34ca72e1b1497b88bbb28cf0aa9e21, entries=100, sequenceid=211, filesize=9.5 K 2024-12-06T16:01:45,780 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for 51b4277372e700eb3a1230f9ab87509e in 151ms, sequenceid=211, compaction requested=true 2024-12-06T16:01:45,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:45,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:45,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-06T16:01:45,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-06T16:01:45,784 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-06T16:01:45,784 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6150 sec 2024-12-06T16:01:45,787 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.6220 sec 2024-12-06T16:01:45,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:45,819 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:01:45,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 51b4277372e700eb3a1230f9ab87509e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-06T16:01:45,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/443fa7d4b967468c89c2b85adca88372 is 50, key is test_row_0/A:col10/1733500905816/Put/seqid=0 2024-12-06T16:01:45,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742131_1307 (size=16931) 2024-12-06T16:01:45,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/443fa7d4b967468c89c2b85adca88372 2024-12-06T16:01:45,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/8dcb15b81eee43f2ba75a29fca744c7b is 50, key is test_row_0/B:col10/1733500905816/Put/seqid=0 2024-12-06T16:01:45,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500965886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:45,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:45,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500965887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:45,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:45,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:45,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:45,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:45,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:45,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:45,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742132_1308 (size=12151) 2024-12-06T16:01:46,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:46,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500966001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:46,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:46,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:46,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:46,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:46,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500966001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:46,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:46,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:46,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:46,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:46,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500966212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:46,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:46,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500966212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:46,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:46,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:46,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:46,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:46,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:46,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:46,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/8dcb15b81eee43f2ba75a29fca744c7b 2024-12-06T16:01:46,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/1129af84a991422981fc3e5f2024913a is 50, key is test_row_0/C:col10/1733500905816/Put/seqid=0 2024-12-06T16:01:46,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742133_1309 (size=12151) 2024-12-06T16:01:46,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/1129af84a991422981fc3e5f2024913a 2024-12-06T16:01:46,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/443fa7d4b967468c89c2b85adca88372 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/443fa7d4b967468c89c2b85adca88372 2024-12-06T16:01:46,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/443fa7d4b967468c89c2b85adca88372, entries=250, sequenceid=222, filesize=16.5 K 2024-12-06T16:01:46,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/8dcb15b81eee43f2ba75a29fca744c7b as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/8dcb15b81eee43f2ba75a29fca744c7b 2024-12-06T16:01:46,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/8dcb15b81eee43f2ba75a29fca744c7b, entries=150, sequenceid=222, filesize=11.9 K 2024-12-06T16:01:46,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/1129af84a991422981fc3e5f2024913a as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/1129af84a991422981fc3e5f2024913a 2024-12-06T16:01:46,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/1129af84a991422981fc3e5f2024913a, entries=150, sequenceid=222, filesize=11.9 K 2024-12-06T16:01:46,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 51b4277372e700eb3a1230f9ab87509e in 514ms, sequenceid=222, compaction requested=true 2024-12-06T16:01:46,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:46,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51b4277372e700eb3a1230f9ab87509e:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:01:46,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:46,333 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T16:01:46,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51b4277372e700eb3a1230f9ab87509e:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:01:46,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:46,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51b4277372e700eb3a1230f9ab87509e:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:01:46,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:46,333 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T16:01:46,335 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53756 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T16:01:46,335 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 51b4277372e700eb3a1230f9ab87509e/A is initiating minor compaction (all files) 2024-12-06T16:01:46,335 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51b4277372e700eb3a1230f9ab87509e/A in TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:46,335 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/6c87bebfa4e941cab80c0df69b629ea1, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/7608eadefda84735afbc320627b633da, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/5655ea36bbaa4c5fac501b1da6ea79fc, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/443fa7d4b967468c89c2b85adca88372] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp, totalSize=52.5 K 2024-12-06T16:01:46,335 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46586 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T16:01:46,335 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c87bebfa4e941cab80c0df69b629ea1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733500902162 2024-12-06T16:01:46,336 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 51b4277372e700eb3a1230f9ab87509e/B is initiating minor compaction (all files) 2024-12-06T16:01:46,336 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51b4277372e700eb3a1230f9ab87509e/B in TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:46,336 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/4e174896b8d240c9bb9155141ee15fb9, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/9cff1684d47a410f93ae4a3fe4b571a0, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/5214392df5b74a87a6a35745ae92f1bf, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/8dcb15b81eee43f2ba75a29fca744c7b] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp, totalSize=45.5 K 2024-12-06T16:01:46,336 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7608eadefda84735afbc320627b633da, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733500903345 2024-12-06T16:01:46,336 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5655ea36bbaa4c5fac501b1da6ea79fc, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733500904658 2024-12-06T16:01:46,336 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e174896b8d240c9bb9155141ee15fb9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733500902162 2024-12-06T16:01:46,337 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 443fa7d4b967468c89c2b85adca88372, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733500905807 2024-12-06T16:01:46,337 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cff1684d47a410f93ae4a3fe4b571a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733500903345 2024-12-06T16:01:46,337 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 5214392df5b74a87a6a35745ae92f1bf, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733500904658 2024-12-06T16:01:46,338 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dcb15b81eee43f2ba75a29fca744c7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733500905812 2024-12-06T16:01:46,356 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51b4277372e700eb3a1230f9ab87509e#A#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:46,357 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/c17864e13b1f4e40be56e4596bf14831 is 50, key is test_row_0/A:col10/1733500905816/Put/seqid=0 2024-12-06T16:01:46,370 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51b4277372e700eb3a1230f9ab87509e#B#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:46,371 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/f2828fc5e54d41ac8dacf11ab1714046 is 50, key is test_row_0/B:col10/1733500905816/Put/seqid=0 2024-12-06T16:01:46,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742134_1310 (size=12663) 2024-12-06T16:01:46,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742135_1311 (size=12663) 2024-12-06T16:01:46,415 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/f2828fc5e54d41ac8dacf11ab1714046 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/f2828fc5e54d41ac8dacf11ab1714046 2024-12-06T16:01:46,422 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 51b4277372e700eb3a1230f9ab87509e/B of 51b4277372e700eb3a1230f9ab87509e into f2828fc5e54d41ac8dacf11ab1714046(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:46,422 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:46,422 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., storeName=51b4277372e700eb3a1230f9ab87509e/B, priority=12, startTime=1733500906333; duration=0sec 2024-12-06T16:01:46,423 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:46,423 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51b4277372e700eb3a1230f9ab87509e:B 2024-12-06T16:01:46,423 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T16:01:46,425 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46586 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T16:01:46,425 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 51b4277372e700eb3a1230f9ab87509e/C is initiating minor compaction (all files) 2024-12-06T16:01:46,425 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51b4277372e700eb3a1230f9ab87509e/C in TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:46,425 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/985fce4699bd412a88572535386c7c09, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/dd56a78d708643e6892a679ecd3d1aeb, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/5e34ca72e1b1497b88bbb28cf0aa9e21, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/1129af84a991422981fc3e5f2024913a] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp, totalSize=45.5 K 2024-12-06T16:01:46,425 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 985fce4699bd412a88572535386c7c09, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733500902162 2024-12-06T16:01:46,426 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting dd56a78d708643e6892a679ecd3d1aeb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733500903345 2024-12-06T16:01:46,426 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e34ca72e1b1497b88bbb28cf0aa9e21, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733500904658 2024-12-06T16:01:46,426 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 1129af84a991422981fc3e5f2024913a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733500905812 2024-12-06T16:01:46,438 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51b4277372e700eb3a1230f9ab87509e#C#compaction#263 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:46,439 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/0b437ec3634c4824af7ca3f09a87227b is 50, key is test_row_0/C:col10/1733500905816/Put/seqid=0 2024-12-06T16:01:46,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742136_1312 (size=12663) 2024-12-06T16:01:46,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:46,526 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:01:46,527 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 51b4277372e700eb3a1230f9ab87509e 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-06T16:01:46,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/53e7db616c1b4865a364f5eceecff382 is 50, key is test_row_0/A:col10/1733500905884/Put/seqid=0 2024-12-06T16:01:46,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742137_1313 (size=14541) 2024-12-06T16:01:46,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/53e7db616c1b4865a364f5eceecff382 2024-12-06T16:01:46,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:46,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500966547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:46,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500966550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:46,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:46,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:46,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:46,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:46,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:46,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:46,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/1247cb9b83154f24bdbf7bde819eb121 is 50, key is test_row_0/B:col10/1733500905884/Put/seqid=0 2024-12-06T16:01:46,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742138_1314 (size=12151) 2024-12-06T16:01:46,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:46,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500966662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:46,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:46,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:46,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:46,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:46,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500966662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:46,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:46,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:46,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:46,790 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/c17864e13b1f4e40be56e4596bf14831 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/c17864e13b1f4e40be56e4596bf14831 2024-12-06T16:01:46,796 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 51b4277372e700eb3a1230f9ab87509e/A of 51b4277372e700eb3a1230f9ab87509e into c17864e13b1f4e40be56e4596bf14831(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:46,796 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:46,796 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., storeName=51b4277372e700eb3a1230f9ab87509e/A, priority=12, startTime=1733500906333; duration=0sec 2024-12-06T16:01:46,797 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:46,797 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51b4277372e700eb3a1230f9ab87509e:A 2024-12-06T16:01:46,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:46,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500966872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:46,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:46,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:46,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:46,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500966872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:46,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:46,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:46,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:46,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:46,899 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/0b437ec3634c4824af7ca3f09a87227b as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/0b437ec3634c4824af7ca3f09a87227b 2024-12-06T16:01:46,906 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 51b4277372e700eb3a1230f9ab87509e/C of 51b4277372e700eb3a1230f9ab87509e into 0b437ec3634c4824af7ca3f09a87227b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:46,906 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:46,907 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., storeName=51b4277372e700eb3a1230f9ab87509e/C, priority=12, startTime=1733500906333; duration=0sec 2024-12-06T16:01:46,907 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:46,907 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51b4277372e700eb3a1230f9ab87509e:C 2024-12-06T16:01:46,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/1247cb9b83154f24bdbf7bde819eb121 2024-12-06T16:01:46,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/3262d57c1b484679bf248a3c269c6589 is 50, key is test_row_0/C:col10/1733500905884/Put/seqid=0 2024-12-06T16:01:46,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742139_1315 (size=12151) 2024-12-06T16:01:46,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/3262d57c1b484679bf248a3c269c6589 2024-12-06T16:01:47,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/53e7db616c1b4865a364f5eceecff382 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/53e7db616c1b4865a364f5eceecff382 2024-12-06T16:01:47,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/53e7db616c1b4865a364f5eceecff382, entries=200, sequenceid=250, filesize=14.2 K 2024-12-06T16:01:47,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/1247cb9b83154f24bdbf7bde819eb121 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/1247cb9b83154f24bdbf7bde819eb121 2024-12-06T16:01:47,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/1247cb9b83154f24bdbf7bde819eb121, entries=150, sequenceid=250, filesize=11.9 K 2024-12-06T16:01:47,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/3262d57c1b484679bf248a3c269c6589 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/3262d57c1b484679bf248a3c269c6589 2024-12-06T16:01:47,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/3262d57c1b484679bf248a3c269c6589, entries=150, sequenceid=250, filesize=11.9 K 2024-12-06T16:01:47,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 51b4277372e700eb3a1230f9ab87509e in 502ms, sequenceid=250, compaction requested=false 2024-12-06T16:01:47,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:47,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:47,186 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:01:47,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 51b4277372e700eb3a1230f9ab87509e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-06T16:01:47,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/9d0adce57946467ba8dd744f2ab61b58 is 50, key is test_row_0/A:col10/1733500907185/Put/seqid=0 2024-12-06T16:01:47,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742140_1316 (size=14691) 2024-12-06T16:01:47,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:47,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500967245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:47,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:47,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:47,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:47,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:47,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500967250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:47,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:47,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:47,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:47,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-06T16:01:47,333 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:01:47,333 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:01:47,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-06T16:01:47,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-06T16:01:47,336 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:01:47,336 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:01:47,337 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:01:47,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:47,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:47,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500967362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:47,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500967362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:47,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:47,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:47,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:47,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:47,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:47,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:47,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-06T16:01:47,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-06T16:01:47,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:47,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:47,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:47,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:47,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:47,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:47,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:47,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500967522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:47,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:47,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500967522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:47,526 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 85bef17d9292,46105,1733500827732 for 'test_row_0' in 51b4277372e700eb3a1230f9ab87509e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T16:01:47,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:47,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:47,526 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 85bef17d9292,46105,1733500827732 for 'test_row_1' in 51b4277372e700eb3a1230f9ab87509e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T16:01:47,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:47,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:47,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500967542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:47,547 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 85bef17d9292,46105,1733500827732 for 'test_row_1' in 51b4277372e700eb3a1230f9ab87509e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T16:01:47,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:47,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:47,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:47,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500967572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:47,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:47,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500967572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:47,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:47,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:47,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:47,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:47,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:47,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/9d0adce57946467ba8dd744f2ab61b58 2024-12-06T16:01:47,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/0c3fa04f14f64c48aa11582d87020ac9 is 50, key is test_row_0/B:col10/1733500907185/Put/seqid=0 2024-12-06T16:01:47,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742141_1317 (size=12251) 2024-12-06T16:01:47,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-06T16:01:47,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:47,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:47,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:47,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:47,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-06T16:01:47,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-06T16:01:47,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:47,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:47,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:47,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:47,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:47,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:47,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500967881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:47,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:47,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:47,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:47,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500967881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:47,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:47,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:47,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:47,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:47,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-06T16:01:47,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:47,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:47,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:47,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:47,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:47,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:47,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-06T16:01:48,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/0c3fa04f14f64c48aa11582d87020ac9 2024-12-06T16:01:48,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/4bd45bdbe5674c6db7ce47792463d7ad is 50, key is test_row_0/C:col10/1733500907185/Put/seqid=0 2024-12-06T16:01:48,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742142_1318 (size=12251) 2024-12-06T16:01:48,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/4bd45bdbe5674c6db7ce47792463d7ad 2024-12-06T16:01:48,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/9d0adce57946467ba8dd744f2ab61b58 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/9d0adce57946467ba8dd744f2ab61b58 2024-12-06T16:01:48,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/9d0adce57946467ba8dd744f2ab61b58, entries=200, sequenceid=264, filesize=14.3 K 2024-12-06T16:01:48,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/0c3fa04f14f64c48aa11582d87020ac9 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/0c3fa04f14f64c48aa11582d87020ac9 2024-12-06T16:01:48,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/0c3fa04f14f64c48aa11582d87020ac9, entries=150, sequenceid=264, filesize=12.0 K 2024-12-06T16:01:48,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/4bd45bdbe5674c6db7ce47792463d7ad as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/4bd45bdbe5674c6db7ce47792463d7ad 2024-12-06T16:01:48,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/4bd45bdbe5674c6db7ce47792463d7ad, entries=150, sequenceid=264, filesize=12.0 K 2024-12-06T16:01:48,098 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 51b4277372e700eb3a1230f9ab87509e in 912ms, sequenceid=264, compaction requested=true 2024-12-06T16:01:48,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:48,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51b4277372e700eb3a1230f9ab87509e:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:01:48,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:48,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51b4277372e700eb3a1230f9ab87509e:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:01:48,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:48,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51b4277372e700eb3a1230f9ab87509e:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:01:48,098 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:48,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:48,099 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:48,100 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41895 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:48,100 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 51b4277372e700eb3a1230f9ab87509e/A is initiating minor compaction (all files) 2024-12-06T16:01:48,100 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51b4277372e700eb3a1230f9ab87509e/A in TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:48,100 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/c17864e13b1f4e40be56e4596bf14831, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/53e7db616c1b4865a364f5eceecff382, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/9d0adce57946467ba8dd744f2ab61b58] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp, totalSize=40.9 K 2024-12-06T16:01:48,100 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:48,100 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting c17864e13b1f4e40be56e4596bf14831, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733500905812 2024-12-06T16:01:48,100 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 51b4277372e700eb3a1230f9ab87509e/B is initiating minor compaction (all files) 2024-12-06T16:01:48,100 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51b4277372e700eb3a1230f9ab87509e/B in TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:48,101 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/f2828fc5e54d41ac8dacf11ab1714046, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/1247cb9b83154f24bdbf7bde819eb121, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/0c3fa04f14f64c48aa11582d87020ac9] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp, totalSize=36.2 K 2024-12-06T16:01:48,102 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting f2828fc5e54d41ac8dacf11ab1714046, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733500905812 2024-12-06T16:01:48,102 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53e7db616c1b4865a364f5eceecff382, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733500905872 2024-12-06T16:01:48,103 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 1247cb9b83154f24bdbf7bde819eb121, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733500905884 2024-12-06T16:01:48,103 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d0adce57946467ba8dd744f2ab61b58, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733500906539 2024-12-06T16:01:48,103 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c3fa04f14f64c48aa11582d87020ac9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733500906539 2024-12-06T16:01:48,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-06T16:01:48,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:48,112 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 51b4277372e700eb3a1230f9ab87509e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-06T16:01:48,118 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51b4277372e700eb3a1230f9ab87509e#A#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:48,121 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/82b1a32e011a459e8cb4170b596bb838 is 50, key is test_row_0/A:col10/1733500907185/Put/seqid=0 2024-12-06T16:01:48,126 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51b4277372e700eb3a1230f9ab87509e#B#compaction#271 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:48,126 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/2d98b9dc2e7b4388a9d295d9191d2efe is 50, key is test_row_0/B:col10/1733500907185/Put/seqid=0 2024-12-06T16:01:48,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/d4261daea5474d2596ed3cbeffd1ae97 is 50, key is test_row_0/A:col10/1733500907222/Put/seqid=0 2024-12-06T16:01:48,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742145_1321 (size=12301) 2024-12-06T16:01:48,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742143_1319 (size=12865) 2024-12-06T16:01:48,213 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/82b1a32e011a459e8cb4170b596bb838 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/82b1a32e011a459e8cb4170b596bb838 2024-12-06T16:01:48,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742144_1320 (size=12865) 2024-12-06T16:01:48,245 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/2d98b9dc2e7b4388a9d295d9191d2efe as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/2d98b9dc2e7b4388a9d295d9191d2efe 2024-12-06T16:01:48,252 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51b4277372e700eb3a1230f9ab87509e/A of 51b4277372e700eb3a1230f9ab87509e into 82b1a32e011a459e8cb4170b596bb838(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:48,252 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:48,252 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., storeName=51b4277372e700eb3a1230f9ab87509e/A, priority=13, startTime=1733500908098; duration=0sec 2024-12-06T16:01:48,252 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:48,252 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51b4277372e700eb3a1230f9ab87509e:A 2024-12-06T16:01:48,252 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:48,255 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51b4277372e700eb3a1230f9ab87509e/B of 51b4277372e700eb3a1230f9ab87509e into 2d98b9dc2e7b4388a9d295d9191d2efe(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:48,255 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:48,255 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., storeName=51b4277372e700eb3a1230f9ab87509e/B, priority=13, startTime=1733500908098; duration=0sec 2024-12-06T16:01:48,255 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:48,255 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51b4277372e700eb3a1230f9ab87509e:B 2024-12-06T16:01:48,256 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:48,256 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 51b4277372e700eb3a1230f9ab87509e/C is initiating minor compaction (all files) 2024-12-06T16:01:48,256 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51b4277372e700eb3a1230f9ab87509e/C in TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:48,256 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/0b437ec3634c4824af7ca3f09a87227b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/3262d57c1b484679bf248a3c269c6589, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/4bd45bdbe5674c6db7ce47792463d7ad] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp, totalSize=36.2 K 2024-12-06T16:01:48,256 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b437ec3634c4824af7ca3f09a87227b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733500905812 2024-12-06T16:01:48,257 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3262d57c1b484679bf248a3c269c6589, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733500905884 2024-12-06T16:01:48,257 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bd45bdbe5674c6db7ce47792463d7ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733500906539 2024-12-06T16:01:48,265 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51b4277372e700eb3a1230f9ab87509e#C#compaction#273 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:48,266 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/6df4f1bf5ec241fe8cb95c6e8bf6bcc0 is 50, key is test_row_0/C:col10/1733500907185/Put/seqid=0 2024-12-06T16:01:48,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742146_1322 (size=12865) 2024-12-06T16:01:48,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:48,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:48,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:48,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500968410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:48,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:48,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500968412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:48,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:48,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:48,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-06T16:01:48,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:48,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500968522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:48,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:48,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:48,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:48,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500968522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:48,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:48,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:48,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:48,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:48,603 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/d4261daea5474d2596ed3cbeffd1ae97 2024-12-06T16:01:48,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/d01ffd61d40d4555b397ccf570a3a719 is 50, key is test_row_0/B:col10/1733500907222/Put/seqid=0 2024-12-06T16:01:48,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742147_1323 (size=12301) 2024-12-06T16:01:48,701 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/6df4f1bf5ec241fe8cb95c6e8bf6bcc0 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/6df4f1bf5ec241fe8cb95c6e8bf6bcc0 2024-12-06T16:01:48,711 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51b4277372e700eb3a1230f9ab87509e/C of 51b4277372e700eb3a1230f9ab87509e into 6df4f1bf5ec241fe8cb95c6e8bf6bcc0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:48,712 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:48,712 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., storeName=51b4277372e700eb3a1230f9ab87509e/C, priority=13, startTime=1733500908098; duration=0sec 2024-12-06T16:01:48,712 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:48,712 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51b4277372e700eb3a1230f9ab87509e:C 2024-12-06T16:01:48,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:48,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500968732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:48,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:48,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500968732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:49,034 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/d01ffd61d40d4555b397ccf570a3a719 2024-12-06T16:01:49,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:49,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500969044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:49,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/0b7e6c5a089f4e7492e1275df2ecd290 is 50, key is test_row_0/C:col10/1733500907222/Put/seqid=0 2024-12-06T16:01:49,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:49,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:49,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:49,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:49,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500969044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:49,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:49,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:49,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:49,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742148_1324 (size=12301) 2024-12-06T16:01:49,461 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/0b7e6c5a089f4e7492e1275df2ecd290 2024-12-06T16:01:49,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/d4261daea5474d2596ed3cbeffd1ae97 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/d4261daea5474d2596ed3cbeffd1ae97 2024-12-06T16:01:49,470 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/d4261daea5474d2596ed3cbeffd1ae97, entries=150, sequenceid=288, filesize=12.0 K 2024-12-06T16:01:49,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/d01ffd61d40d4555b397ccf570a3a719 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/d01ffd61d40d4555b397ccf570a3a719 2024-12-06T16:01:49,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-06T16:01:49,485 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/d01ffd61d40d4555b397ccf570a3a719, entries=150, sequenceid=288, filesize=12.0 K 2024-12-06T16:01:49,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/0b7e6c5a089f4e7492e1275df2ecd290 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/0b7e6c5a089f4e7492e1275df2ecd290 2024-12-06T16:01:49,504 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/0b7e6c5a089f4e7492e1275df2ecd290, entries=150, sequenceid=288, filesize=12.0 K 2024-12-06T16:01:49,505 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 51b4277372e700eb3a1230f9ab87509e in 1393ms, sequenceid=288, compaction requested=false 2024-12-06T16:01:49,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:49,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:49,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-06T16:01:49,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-06T16:01:49,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-06T16:01:49,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1700 sec 2024-12-06T16:01:49,513 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.1780 sec 2024-12-06T16:01:49,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:49,555 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:01:49,555 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 51b4277372e700eb3a1230f9ab87509e 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-06T16:01:49,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/ffd0c99559904f2b9bdc92678a31e008 is 50, key is test_row_0/A:col10/1733500908410/Put/seqid=0 2024-12-06T16:01:49,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742149_1325 (size=12301) 2024-12-06T16:01:49,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:49,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500969609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:49,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:49,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:49,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:49,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:49,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500969612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:49,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500969722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:49,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:49,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500969722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:49,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:49,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:49,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:49,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:49,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:49,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:49,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500969931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:49,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:49,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:49,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:49,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:49,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500969931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:49,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:49,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:49,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:49,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/ffd0c99559904f2b9bdc92678a31e008 2024-12-06T16:01:49,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/4096f9cab52e4466928561cd3c539b3a is 50, key is test_row_0/B:col10/1733500908410/Put/seqid=0 2024-12-06T16:01:49,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742150_1326 (size=12301) 2024-12-06T16:01:49,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/4096f9cab52e4466928561cd3c539b3a 2024-12-06T16:01:49,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/20f51eb663f54fe28ce8ccb411858eab is 50, key is test_row_0/C:col10/1733500908410/Put/seqid=0 2024-12-06T16:01:50,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742151_1327 (size=12301) 2024-12-06T16:01:50,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/20f51eb663f54fe28ce8ccb411858eab 2024-12-06T16:01:50,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/ffd0c99559904f2b9bdc92678a31e008 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/ffd0c99559904f2b9bdc92678a31e008 2024-12-06T16:01:50,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/ffd0c99559904f2b9bdc92678a31e008, entries=150, sequenceid=304, filesize=12.0 K 2024-12-06T16:01:50,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/4096f9cab52e4466928561cd3c539b3a as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/4096f9cab52e4466928561cd3c539b3a 2024-12-06T16:01:50,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/4096f9cab52e4466928561cd3c539b3a, entries=150, sequenceid=304, filesize=12.0 K 2024-12-06T16:01:50,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/20f51eb663f54fe28ce8ccb411858eab as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/20f51eb663f54fe28ce8ccb411858eab 2024-12-06T16:01:50,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/20f51eb663f54fe28ce8ccb411858eab, entries=150, sequenceid=304, filesize=12.0 K 2024-12-06T16:01:50,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 51b4277372e700eb3a1230f9ab87509e in 477ms, sequenceid=304, compaction requested=true 2024-12-06T16:01:50,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:50,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51b4277372e700eb3a1230f9ab87509e:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:01:50,033 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:50,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:50,033 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:50,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51b4277372e700eb3a1230f9ab87509e:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:01:50,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:50,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51b4277372e700eb3a1230f9ab87509e:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:01:50,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:50,034 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:50,034 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 51b4277372e700eb3a1230f9ab87509e/A is initiating minor compaction (all files) 2024-12-06T16:01:50,034 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51b4277372e700eb3a1230f9ab87509e/A in TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:50,034 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/82b1a32e011a459e8cb4170b596bb838, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/d4261daea5474d2596ed3cbeffd1ae97, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/ffd0c99559904f2b9bdc92678a31e008] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp, totalSize=36.6 K 2024-12-06T16:01:50,034 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:50,034 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 51b4277372e700eb3a1230f9ab87509e/B is initiating minor compaction (all files) 2024-12-06T16:01:50,034 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82b1a32e011a459e8cb4170b596bb838, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733500906539 2024-12-06T16:01:50,034 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51b4277372e700eb3a1230f9ab87509e/B in TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:50,035 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/2d98b9dc2e7b4388a9d295d9191d2efe, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/d01ffd61d40d4555b397ccf570a3a719, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/4096f9cab52e4466928561cd3c539b3a] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp, totalSize=36.6 K 2024-12-06T16:01:50,035 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4261daea5474d2596ed3cbeffd1ae97, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733500907222 2024-12-06T16:01:50,036 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d98b9dc2e7b4388a9d295d9191d2efe, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733500906539 2024-12-06T16:01:50,036 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting d01ffd61d40d4555b397ccf570a3a719, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733500907222 2024-12-06T16:01:50,036 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffd0c99559904f2b9bdc92678a31e008, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733500908408 2024-12-06T16:01:50,036 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 4096f9cab52e4466928561cd3c539b3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733500908408 2024-12-06T16:01:50,046 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51b4277372e700eb3a1230f9ab87509e#B#compaction#279 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:50,047 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/8158b0edd1094777a345285a53671c65 is 50, key is test_row_0/B:col10/1733500908410/Put/seqid=0 2024-12-06T16:01:50,054 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51b4277372e700eb3a1230f9ab87509e#A#compaction#280 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:50,055 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/e165deddab9944a8b40e2107094f4ac0 is 50, key is test_row_0/A:col10/1733500908410/Put/seqid=0 2024-12-06T16:01:50,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742153_1329 (size=13017) 2024-12-06T16:01:50,075 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/e165deddab9944a8b40e2107094f4ac0 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/e165deddab9944a8b40e2107094f4ac0 2024-12-06T16:01:50,082 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51b4277372e700eb3a1230f9ab87509e/A of 51b4277372e700eb3a1230f9ab87509e into e165deddab9944a8b40e2107094f4ac0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:50,082 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:50,082 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., storeName=51b4277372e700eb3a1230f9ab87509e/A, priority=13, startTime=1733500910033; duration=0sec 2024-12-06T16:01:50,082 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:50,083 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51b4277372e700eb3a1230f9ab87509e:A 2024-12-06T16:01:50,083 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:50,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742152_1328 (size=13017) 2024-12-06T16:01:50,085 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:50,085 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 51b4277372e700eb3a1230f9ab87509e/C is initiating minor compaction (all files) 2024-12-06T16:01:50,085 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51b4277372e700eb3a1230f9ab87509e/C in TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:50,086 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/6df4f1bf5ec241fe8cb95c6e8bf6bcc0, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/0b7e6c5a089f4e7492e1275df2ecd290, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/20f51eb663f54fe28ce8ccb411858eab] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp, totalSize=36.6 K 2024-12-06T16:01:50,087 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6df4f1bf5ec241fe8cb95c6e8bf6bcc0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733500906539 2024-12-06T16:01:50,088 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b7e6c5a089f4e7492e1275df2ecd290, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733500907222 2024-12-06T16:01:50,088 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20f51eb663f54fe28ce8ccb411858eab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733500908408 2024-12-06T16:01:50,091 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/8158b0edd1094777a345285a53671c65 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/8158b0edd1094777a345285a53671c65 2024-12-06T16:01:50,102 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51b4277372e700eb3a1230f9ab87509e/B of 51b4277372e700eb3a1230f9ab87509e into 8158b0edd1094777a345285a53671c65(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:50,102 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:50,102 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., storeName=51b4277372e700eb3a1230f9ab87509e/B, priority=13, startTime=1733500910033; duration=0sec 2024-12-06T16:01:50,102 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:50,102 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51b4277372e700eb3a1230f9ab87509e:B 2024-12-06T16:01:50,108 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51b4277372e700eb3a1230f9ab87509e#C#compaction#281 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:50,108 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/e8836b40b79e4d6bb7a39304562d1ba8 is 50, key is test_row_0/C:col10/1733500908410/Put/seqid=0 2024-12-06T16:01:50,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742154_1330 (size=13017) 2024-12-06T16:01:50,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:50,245 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:01:50,245 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 51b4277372e700eb3a1230f9ab87509e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-06T16:01:50,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/d40cb36c901e4663b0252b63ac5a524b is 50, key is test_row_0/A:col10/1733500910245/Put/seqid=0 2024-12-06T16:01:50,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742155_1331 (size=14741) 2024-12-06T16:01:50,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/d40cb36c901e4663b0252b63ac5a524b 2024-12-06T16:01:50,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:50,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500970273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:50,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:50,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500970275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:50,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:50,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:50,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:50,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:50,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:50,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:50,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/96ff259437964f06a6a558d8631fbb6b is 50, key is test_row_0/B:col10/1733500910245/Put/seqid=0 2024-12-06T16:01:50,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742156_1332 (size=12301) 2024-12-06T16:01:50,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/96ff259437964f06a6a558d8631fbb6b 2024-12-06T16:01:50,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/04c172ad2b3248fd983c87e3abc88a08 is 50, key is test_row_0/C:col10/1733500910245/Put/seqid=0 2024-12-06T16:01:50,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742157_1333 (size=12301) 2024-12-06T16:01:50,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/04c172ad2b3248fd983c87e3abc88a08 2024-12-06T16:01:50,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/d40cb36c901e4663b0252b63ac5a524b as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/d40cb36c901e4663b0252b63ac5a524b 2024-12-06T16:01:50,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/d40cb36c901e4663b0252b63ac5a524b, entries=200, sequenceid=331, filesize=14.4 K 2024-12-06T16:01:50,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/96ff259437964f06a6a558d8631fbb6b as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/96ff259437964f06a6a558d8631fbb6b 2024-12-06T16:01:50,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/96ff259437964f06a6a558d8631fbb6b, entries=150, sequenceid=331, filesize=12.0 K 2024-12-06T16:01:50,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/04c172ad2b3248fd983c87e3abc88a08 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/04c172ad2b3248fd983c87e3abc88a08 2024-12-06T16:01:50,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/04c172ad2b3248fd983c87e3abc88a08, entries=150, sequenceid=331, filesize=12.0 K 2024-12-06T16:01:50,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 51b4277372e700eb3a1230f9ab87509e in 93ms, sequenceid=331, compaction requested=false 2024-12-06T16:01:50,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:50,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:50,387 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:01:50,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 51b4277372e700eb3a1230f9ab87509e 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-06T16:01:50,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/6ee96f02ab2f49eda64291d6d97f4d09 is 50, key is test_row_0/A:col10/1733500910387/Put/seqid=0 2024-12-06T16:01:50,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742158_1334 (size=14741) 2024-12-06T16:01:50,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:50,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500970476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:50,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:50,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:50,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:50,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:50,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500970482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:50,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:50,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:50,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:50,538 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/e8836b40b79e4d6bb7a39304562d1ba8 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/e8836b40b79e4d6bb7a39304562d1ba8 2024-12-06T16:01:50,553 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51b4277372e700eb3a1230f9ab87509e/C of 51b4277372e700eb3a1230f9ab87509e into e8836b40b79e4d6bb7a39304562d1ba8(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:50,553 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:50,553 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., storeName=51b4277372e700eb3a1230f9ab87509e/C, priority=13, startTime=1733500910033; duration=0sec 2024-12-06T16:01:50,553 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:50,554 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51b4277372e700eb3a1230f9ab87509e:C 2024-12-06T16:01:50,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:50,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500970592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:50,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:50,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:50,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:50,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500970592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:50,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:50,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:50,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:50,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:50,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500970802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:50,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500970802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:50,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:50,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:50,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:50,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:50,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:50,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:50,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/6ee96f02ab2f49eda64291d6d97f4d09 2024-12-06T16:01:50,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/a969cac3bc194692b996f2f57769e777 is 50, key is test_row_0/B:col10/1733500910387/Put/seqid=0 2024-12-06T16:01:50,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742159_1335 (size=12301) 2024-12-06T16:01:50,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/a969cac3bc194692b996f2f57769e777 2024-12-06T16:01:50,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/32063ce73792434db3ee4a2a4ff6aee3 is 50, key is test_row_0/C:col10/1733500910387/Put/seqid=0 2024-12-06T16:01:50,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742160_1336 (size=12301) 2024-12-06T16:01:50,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/32063ce73792434db3ee4a2a4ff6aee3 2024-12-06T16:01:50,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/6ee96f02ab2f49eda64291d6d97f4d09 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/6ee96f02ab2f49eda64291d6d97f4d09 2024-12-06T16:01:50,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/6ee96f02ab2f49eda64291d6d97f4d09, entries=200, sequenceid=344, filesize=14.4 K 2024-12-06T16:01:50,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/a969cac3bc194692b996f2f57769e777 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/a969cac3bc194692b996f2f57769e777 2024-12-06T16:01:50,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/a969cac3bc194692b996f2f57769e777, entries=150, sequenceid=344, filesize=12.0 K 2024-12-06T16:01:50,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/32063ce73792434db3ee4a2a4ff6aee3 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/32063ce73792434db3ee4a2a4ff6aee3 2024-12-06T16:01:50,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/32063ce73792434db3ee4a2a4ff6aee3, entries=150, sequenceid=344, filesize=12.0 K 2024-12-06T16:01:50,910 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 51b4277372e700eb3a1230f9ab87509e in 523ms, sequenceid=344, compaction requested=true 2024-12-06T16:01:50,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:50,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51b4277372e700eb3a1230f9ab87509e:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:01:50,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:50,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51b4277372e700eb3a1230f9ab87509e:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:01:50,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:50,911 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:50,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51b4277372e700eb3a1230f9ab87509e:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:01:50,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:50,911 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:50,912 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42499 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:50,912 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 51b4277372e700eb3a1230f9ab87509e/A is initiating minor compaction (all files) 2024-12-06T16:01:50,912 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51b4277372e700eb3a1230f9ab87509e/A in TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:50,912 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/e165deddab9944a8b40e2107094f4ac0, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/d40cb36c901e4663b0252b63ac5a524b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/6ee96f02ab2f49eda64291d6d97f4d09] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp, totalSize=41.5 K 2024-12-06T16:01:50,912 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:50,912 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 51b4277372e700eb3a1230f9ab87509e/B is initiating minor compaction (all files) 2024-12-06T16:01:50,912 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51b4277372e700eb3a1230f9ab87509e/B in TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:50,912 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/8158b0edd1094777a345285a53671c65, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/96ff259437964f06a6a558d8631fbb6b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/a969cac3bc194692b996f2f57769e777] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp, totalSize=36.7 K 2024-12-06T16:01:50,913 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting e165deddab9944a8b40e2107094f4ac0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733500908408 2024-12-06T16:01:50,913 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 8158b0edd1094777a345285a53671c65, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733500908408 2024-12-06T16:01:50,913 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting d40cb36c901e4663b0252b63ac5a524b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1733500909599 2024-12-06T16:01:50,913 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 96ff259437964f06a6a558d8631fbb6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1733500909599 2024-12-06T16:01:50,915 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ee96f02ab2f49eda64291d6d97f4d09, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1733500910258 2024-12-06T16:01:50,915 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting a969cac3bc194692b996f2f57769e777, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1733500910258 2024-12-06T16:01:50,926 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51b4277372e700eb3a1230f9ab87509e#B#compaction#288 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:50,927 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/ebb2c2a24eb14fb8b0d9c46835df6e1e is 50, key is test_row_0/B:col10/1733500910387/Put/seqid=0 2024-12-06T16:01:50,938 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51b4277372e700eb3a1230f9ab87509e#A#compaction#289 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:50,939 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/d16d883584c144f5bb43c636a4f27a30 is 50, key is test_row_0/A:col10/1733500910387/Put/seqid=0 2024-12-06T16:01:50,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742161_1337 (size=13119) 2024-12-06T16:01:50,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742162_1338 (size=13119) 2024-12-06T16:01:51,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:51,117 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:01:51,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 51b4277372e700eb3a1230f9ab87509e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-06T16:01:51,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/32dbacf8ce8b4cb68fe950978d54872d is 50, key is test_row_0/A:col10/1733500911116/Put/seqid=0 2024-12-06T16:01:51,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742163_1339 (size=14741) 2024-12-06T16:01:51,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500971139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:51,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:51,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:51,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500971140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:51,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:51,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:51,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:51,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:51,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500971251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:51,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:51,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500971252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:51,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:51,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:51,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:51,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:51,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:51,347 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/ebb2c2a24eb14fb8b0d9c46835df6e1e as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/ebb2c2a24eb14fb8b0d9c46835df6e1e 2024-12-06T16:01:51,349 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/d16d883584c144f5bb43c636a4f27a30 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/d16d883584c144f5bb43c636a4f27a30 2024-12-06T16:01:51,352 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51b4277372e700eb3a1230f9ab87509e/B of 51b4277372e700eb3a1230f9ab87509e into ebb2c2a24eb14fb8b0d9c46835df6e1e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:51,352 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:51,352 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., storeName=51b4277372e700eb3a1230f9ab87509e/B, priority=13, startTime=1733500910911; duration=0sec 2024-12-06T16:01:51,352 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:51,352 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51b4277372e700eb3a1230f9ab87509e:B 2024-12-06T16:01:51,352 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:51,353 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:51,353 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 51b4277372e700eb3a1230f9ab87509e/C is initiating minor compaction (all files) 2024-12-06T16:01:51,353 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51b4277372e700eb3a1230f9ab87509e/C in TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:51,353 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/e8836b40b79e4d6bb7a39304562d1ba8, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/04c172ad2b3248fd983c87e3abc88a08, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/32063ce73792434db3ee4a2a4ff6aee3] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp, totalSize=36.7 K 2024-12-06T16:01:51,354 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting e8836b40b79e4d6bb7a39304562d1ba8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733500908408 2024-12-06T16:01:51,355 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 04c172ad2b3248fd983c87e3abc88a08, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1733500909599 2024-12-06T16:01:51,355 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51b4277372e700eb3a1230f9ab87509e/A of 51b4277372e700eb3a1230f9ab87509e into d16d883584c144f5bb43c636a4f27a30(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:51,355 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:51,355 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., storeName=51b4277372e700eb3a1230f9ab87509e/A, priority=13, startTime=1733500910911; duration=0sec 2024-12-06T16:01:51,355 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:51,355 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51b4277372e700eb3a1230f9ab87509e:A 2024-12-06T16:01:51,356 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 32063ce73792434db3ee4a2a4ff6aee3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1733500910258 2024-12-06T16:01:51,362 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51b4277372e700eb3a1230f9ab87509e#C#compaction#291 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:51,362 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/77e64d3a33be4222bc60e9ab15f46fd7 is 50, key is test_row_0/C:col10/1733500910387/Put/seqid=0 2024-12-06T16:01:51,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742164_1340 (size=13119) 2024-12-06T16:01:51,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500971461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:51,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500971462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:51,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:51,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:51,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:51,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:51,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:51,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-06T16:01:51,502 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:01:51,503 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:01:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-06T16:01:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-06T16:01:51,505 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:01:51,506 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:01:51,506 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:01:51,527 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/32dbacf8ce8b4cb68fe950978d54872d 2024-12-06T16:01:51,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/f02941321e03440b970979db92ee39d3 is 50, key is test_row_0/B:col10/1733500911116/Put/seqid=0 2024-12-06T16:01:51,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742165_1341 (size=12301) 2024-12-06T16:01:51,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/f02941321e03440b970979db92ee39d3 2024-12-06T16:01:51,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59442 deadline: 1733500971542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:51,546 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 85bef17d9292,46105,1733500827732 for 'test_row_1' in 51b4277372e700eb3a1230f9ab87509e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T16:01:51,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:51,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:51,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:51,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/e93afe066202466bb744da22b460390c is 50, key is test_row_0/C:col10/1733500911116/Put/seqid=0 2024-12-06T16:01:51,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742166_1342 (size=12301) 2024-12-06T16:01:51,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/e93afe066202466bb744da22b460390c 2024-12-06T16:01:51,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/32dbacf8ce8b4cb68fe950978d54872d as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/32dbacf8ce8b4cb68fe950978d54872d 2024-12-06T16:01:51,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/32dbacf8ce8b4cb68fe950978d54872d, entries=200, sequenceid=370, filesize=14.4 K 2024-12-06T16:01:51,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733500971562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:51,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59412 deadline: 1733500971562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:51,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/f02941321e03440b970979db92ee39d3 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/f02941321e03440b970979db92ee39d3 2024-12-06T16:01:51,566 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 85bef17d9292,46105,1733500827732 for 'test_row_1' in 51b4277372e700eb3a1230f9ab87509e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T16:01:51,566 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 85bef17d9292,46105,1733500827732 for 'test_row_0' in 51b4277372e700eb3a1230f9ab87509e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T16:01:51,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:51,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:51,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:51,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:51,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:51,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:51,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/f02941321e03440b970979db92ee39d3, entries=150, sequenceid=370, filesize=12.0 K 2024-12-06T16:01:51,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/e93afe066202466bb744da22b460390c as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/e93afe066202466bb744da22b460390c 2024-12-06T16:01:51,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/e93afe066202466bb744da22b460390c, entries=150, sequenceid=370, filesize=12.0 K 2024-12-06T16:01:51,575 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 51b4277372e700eb3a1230f9ab87509e in 458ms, sequenceid=370, compaction requested=false 2024-12-06T16:01:51,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-06T16:01:51,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-06T16:01:51,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:51,659 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 51b4277372e700eb3a1230f9ab87509e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-06T16:01:51,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/49bbc8a22c924bdf9c14d17f093b6726 is 50, key is test_row_0/A:col10/1733500911138/Put/seqid=0 2024-12-06T16:01:51,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742167_1343 (size=12301) 2024-12-06T16:01:51,669 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/49bbc8a22c924bdf9c14d17f093b6726 2024-12-06T16:01:51,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/c7ae5eb9b5bb4828bf688307031e2154 is 50, key is test_row_0/B:col10/1733500911138/Put/seqid=0 2024-12-06T16:01:51,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742168_1344 (size=12301) 2024-12-06T16:01:51,688 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/c7ae5eb9b5bb4828bf688307031e2154 2024-12-06T16:01:51,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/fb05d8982dc14f71bac1f1af391616db is 50, key is test_row_0/C:col10/1733500911138/Put/seqid=0 2024-12-06T16:01:51,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742169_1345 (size=12301) 2024-12-06T16:01:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:51,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:51,782 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/77e64d3a33be4222bc60e9ab15f46fd7 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/77e64d3a33be4222bc60e9ab15f46fd7 2024-12-06T16:01:51,788 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51b4277372e700eb3a1230f9ab87509e/C of 51b4277372e700eb3a1230f9ab87509e into 77e64d3a33be4222bc60e9ab15f46fd7(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:51,788 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:51,788 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., storeName=51b4277372e700eb3a1230f9ab87509e/C, priority=13, startTime=1733500910911; duration=0sec 2024-12-06T16:01:51,788 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:51,788 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51b4277372e700eb3a1230f9ab87509e:C 2024-12-06T16:01:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-06T16:01:51,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:51,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500971835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:51,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:51,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:51,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:51,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500971837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:51,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:51,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:51,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:51,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:51,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500971942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:51,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:51,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:51,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:51,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:51,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500971952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:51,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:51,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:51,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:52,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-06T16:01:52,135 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/fb05d8982dc14f71bac1f1af391616db 2024-12-06T16:01:52,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/49bbc8a22c924bdf9c14d17f093b6726 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/49bbc8a22c924bdf9c14d17f093b6726 2024-12-06T16:01:52,144 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/49bbc8a22c924bdf9c14d17f093b6726, entries=150, sequenceid=383, filesize=12.0 K 2024-12-06T16:01:52,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/c7ae5eb9b5bb4828bf688307031e2154 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/c7ae5eb9b5bb4828bf688307031e2154 2024-12-06T16:01:52,149 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/c7ae5eb9b5bb4828bf688307031e2154, entries=150, sequenceid=383, filesize=12.0 K 2024-12-06T16:01:52,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/fb05d8982dc14f71bac1f1af391616db as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/fb05d8982dc14f71bac1f1af391616db 2024-12-06T16:01:52,154 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/fb05d8982dc14f71bac1f1af391616db, entries=150, sequenceid=383, filesize=12.0 K 2024-12-06T16:01:52,154 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 51b4277372e700eb3a1230f9ab87509e in 495ms, sequenceid=383, compaction requested=true 2024-12-06T16:01:52,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:52,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:52,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-06T16:01:52,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-06T16:01:52,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:52,158 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:01:52,158 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-06T16:01:52,158 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 650 msec 2024-12-06T16:01:52,158 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 51b4277372e700eb3a1230f9ab87509e 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-06T16:01:52,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 656 msec 2024-12-06T16:01:52,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/b2d796541d344ef7aee886fe2e52f894 is 50, key is test_row_0/A:col10/1733500911822/Put/seqid=0 2024-12-06T16:01:52,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742170_1346 (size=14741) 2024-12-06T16:01:52,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500972176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:52,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500972177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:52,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:52,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:52,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:52,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:52,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:52,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:52,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500972291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:52,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500972291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:52,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500972502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:52,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500972502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:52,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/b2d796541d344ef7aee886fe2e52f894 2024-12-06T16:01:52,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/70456bed0aa1496ab4166fafb6d25626 is 50, key is test_row_0/B:col10/1733500911822/Put/seqid=0 2024-12-06T16:01:52,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742171_1347 (size=12301) 2024-12-06T16:01:52,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/70456bed0aa1496ab4166fafb6d25626 2024-12-06T16:01:52,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/1dee31450b7a4e7e99fb7d7d83def59c is 50, key is test_row_0/C:col10/1733500911822/Put/seqid=0 2024-12-06T16:01:52,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742172_1348 (size=12301) 2024-12-06T16:01:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-06T16:01:52,642 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:01:52,642 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:01:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-06T16:01:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-06T16:01:52,645 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:01:52,646 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:01:52,646 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:01:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-06T16:01:52,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-06T16:01:52,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:52,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:52,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:52,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:52,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:52,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:52,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:52,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500972812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:52,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:52,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:52,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:52,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:52,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500972812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:52,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:52,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:52,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:52,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-06T16:01:52,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:52,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:52,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:52,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:52,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:52,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:01:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-06T16:01:53,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/1dee31450b7a4e7e99fb7d7d83def59c 2024-12-06T16:01:53,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/b2d796541d344ef7aee886fe2e52f894 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/b2d796541d344ef7aee886fe2e52f894 2024-12-06T16:01:53,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/b2d796541d344ef7aee886fe2e52f894, entries=200, sequenceid=410, filesize=14.4 K 2024-12-06T16:01:53,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/70456bed0aa1496ab4166fafb6d25626 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/70456bed0aa1496ab4166fafb6d25626 2024-12-06T16:01:53,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/70456bed0aa1496ab4166fafb6d25626, entries=150, sequenceid=410, filesize=12.0 K 2024-12-06T16:01:53,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/1dee31450b7a4e7e99fb7d7d83def59c as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/1dee31450b7a4e7e99fb7d7d83def59c 2024-12-06T16:01:53,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/1dee31450b7a4e7e99fb7d7d83def59c, entries=150, sequenceid=410, filesize=12.0 K 2024-12-06T16:01:53,025 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 51b4277372e700eb3a1230f9ab87509e in 867ms, sequenceid=410, compaction requested=true 2024-12-06T16:01:53,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:53,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51b4277372e700eb3a1230f9ab87509e:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:01:53,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:53,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51b4277372e700eb3a1230f9ab87509e:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:01:53,026 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T16:01:53,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:53,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51b4277372e700eb3a1230f9ab87509e:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:01:53,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-06T16:01:53,026 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T16:01:53,027 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54902 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T16:01:53,027 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 51b4277372e700eb3a1230f9ab87509e/A is initiating minor compaction (all files) 2024-12-06T16:01:53,027 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51b4277372e700eb3a1230f9ab87509e/A in TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:53,027 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/d16d883584c144f5bb43c636a4f27a30, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/32dbacf8ce8b4cb68fe950978d54872d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/49bbc8a22c924bdf9c14d17f093b6726, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/b2d796541d344ef7aee886fe2e52f894] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp, totalSize=53.6 K 2024-12-06T16:01:53,027 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T16:01:53,028 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 51b4277372e700eb3a1230f9ab87509e/B is initiating minor compaction (all files) 2024-12-06T16:01:53,028 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51b4277372e700eb3a1230f9ab87509e/B in TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:53,028 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/ebb2c2a24eb14fb8b0d9c46835df6e1e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/f02941321e03440b970979db92ee39d3, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/c7ae5eb9b5bb4828bf688307031e2154, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/70456bed0aa1496ab4166fafb6d25626] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp, totalSize=48.8 K 2024-12-06T16:01:53,028 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting d16d883584c144f5bb43c636a4f27a30, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1733500910258 2024-12-06T16:01:53,028 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting ebb2c2a24eb14fb8b0d9c46835df6e1e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1733500910258 2024-12-06T16:01:53,028 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting f02941321e03440b970979db92ee39d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733500910473 2024-12-06T16:01:53,029 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting c7ae5eb9b5bb4828bf688307031e2154, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733500911133 2024-12-06T16:01:53,029 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32dbacf8ce8b4cb68fe950978d54872d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733500910473 2024-12-06T16:01:53,029 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49bbc8a22c924bdf9c14d17f093b6726, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733500911133 2024-12-06T16:01:53,029 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 70456bed0aa1496ab4166fafb6d25626, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1733500911822 2024-12-06T16:01:53,029 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2d796541d344ef7aee886fe2e52f894, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1733500911822 2024-12-06T16:01:53,050 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51b4277372e700eb3a1230f9ab87509e#B#compaction#300 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:53,051 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/606fe6ab655444d5871588cb5b992bc7 is 50, key is test_row_0/B:col10/1733500911822/Put/seqid=0 2024-12-06T16:01:53,054 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51b4277372e700eb3a1230f9ab87509e#A#compaction#301 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:53,055 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/4d19516dd6654002b8af31bfed2f402c is 50, key is test_row_0/A:col10/1733500911822/Put/seqid=0 2024-12-06T16:01:53,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742173_1349 (size=13255) 2024-12-06T16:01:53,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742174_1350 (size=13255) 2024-12-06T16:01:53,076 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/4d19516dd6654002b8af31bfed2f402c as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/4d19516dd6654002b8af31bfed2f402c 2024-12-06T16:01:53,082 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 51b4277372e700eb3a1230f9ab87509e/A of 51b4277372e700eb3a1230f9ab87509e into 4d19516dd6654002b8af31bfed2f402c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:53,082 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:53,082 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., storeName=51b4277372e700eb3a1230f9ab87509e/A, priority=12, startTime=1733500913025; duration=0sec 2024-12-06T16:01:53,083 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:53,083 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51b4277372e700eb3a1230f9ab87509e:A 2024-12-06T16:01:53,083 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T16:01:53,084 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T16:01:53,084 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 51b4277372e700eb3a1230f9ab87509e/C is initiating minor compaction (all files) 2024-12-06T16:01:53,084 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51b4277372e700eb3a1230f9ab87509e/C in TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:53,084 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/77e64d3a33be4222bc60e9ab15f46fd7, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/e93afe066202466bb744da22b460390c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/fb05d8982dc14f71bac1f1af391616db, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/1dee31450b7a4e7e99fb7d7d83def59c] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp, totalSize=48.8 K 2024-12-06T16:01:53,086 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77e64d3a33be4222bc60e9ab15f46fd7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1733500910258 2024-12-06T16:01:53,086 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting e93afe066202466bb744da22b460390c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733500910473 2024-12-06T16:01:53,087 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb05d8982dc14f71bac1f1af391616db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733500911133 2024-12-06T16:01:53,087 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dee31450b7a4e7e99fb7d7d83def59c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1733500911822 2024-12-06T16:01:53,102 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51b4277372e700eb3a1230f9ab87509e#C#compaction#302 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:53,102 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/0f78ac425648446e903593c9609a3411 is 50, key is test_row_0/C:col10/1733500911822/Put/seqid=0 2024-12-06T16:01:53,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-06T16:01:53,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:53,106 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 51b4277372e700eb3a1230f9ab87509e 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-12-06T16:01:53,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/4afbedc7d0c145b88ec5f569339e2129 is 50, key is test_row_0/A:col10/1733500912175/Put/seqid=0 2024-12-06T16:01:53,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742175_1351 (size=13255) 2024-12-06T16:01:53,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742176_1352 (size=12301) 2024-12-06T16:01:53,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-06T16:01:53,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:53,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. as already flushing 2024-12-06T16:01:53,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:53,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500973406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:53,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:53,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:53,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:53,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:53,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500973411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:53,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:53,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:53,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:53,467 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/606fe6ab655444d5871588cb5b992bc7 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/606fe6ab655444d5871588cb5b992bc7 2024-12-06T16:01:53,472 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 51b4277372e700eb3a1230f9ab87509e/B of 51b4277372e700eb3a1230f9ab87509e into 606fe6ab655444d5871588cb5b992bc7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:53,472 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:53,472 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., storeName=51b4277372e700eb3a1230f9ab87509e/B, priority=12, startTime=1733500913026; duration=0sec 2024-12-06T16:01:53,472 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:53,472 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51b4277372e700eb3a1230f9ab87509e:B 2024-12-06T16:01:53,519 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/0f78ac425648446e903593c9609a3411 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/0f78ac425648446e903593c9609a3411 2024-12-06T16:01:53,524 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 51b4277372e700eb3a1230f9ab87509e/C of 51b4277372e700eb3a1230f9ab87509e into 0f78ac425648446e903593c9609a3411(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:53,524 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:53,524 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., storeName=51b4277372e700eb3a1230f9ab87509e/C, priority=12, startTime=1733500913026; duration=0sec 2024-12-06T16:01:53,524 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:53,524 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51b4277372e700eb3a1230f9ab87509e:C 2024-12-06T16:01:53,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:53,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500973522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:53,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:53,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:53,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500973522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:53,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:53,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:53,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:53,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:53,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:53,532 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/4afbedc7d0c145b88ec5f569339e2129 2024-12-06T16:01:53,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/8ed407431cbc4d378d5aa3f731a47736 is 50, key is test_row_0/B:col10/1733500912175/Put/seqid=0 2024-12-06T16:01:53,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742177_1353 (size=12301) 2024-12-06T16:01:53,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:53,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500973732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:53,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:53,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:53,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:53,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:53,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500973732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:53,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:53,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:53,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:53,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-06T16:01:53,943 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/8ed407431cbc4d378d5aa3f731a47736 2024-12-06T16:01:53,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/3dcabfe4411e4a4387ef69881ceddc9a is 50, key is test_row_0/C:col10/1733500912175/Put/seqid=0 2024-12-06T16:01:53,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742178_1354 (size=12301) 2024-12-06T16:01:54,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:54,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500974042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:54,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:54,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:54,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:54,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:54,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500974042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:54,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:54,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:54,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:54,362 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/3dcabfe4411e4a4387ef69881ceddc9a 2024-12-06T16:01:54,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/4afbedc7d0c145b88ec5f569339e2129 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/4afbedc7d0c145b88ec5f569339e2129 2024-12-06T16:01:54,373 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/4afbedc7d0c145b88ec5f569339e2129, entries=150, sequenceid=421, filesize=12.0 K 2024-12-06T16:01:54,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/8ed407431cbc4d378d5aa3f731a47736 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/8ed407431cbc4d378d5aa3f731a47736 2024-12-06T16:01:54,378 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/8ed407431cbc4d378d5aa3f731a47736, entries=150, sequenceid=421, filesize=12.0 K 2024-12-06T16:01:54,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/3dcabfe4411e4a4387ef69881ceddc9a as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/3dcabfe4411e4a4387ef69881ceddc9a 2024-12-06T16:01:54,385 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/3dcabfe4411e4a4387ef69881ceddc9a, entries=150, sequenceid=421, filesize=12.0 K 2024-12-06T16:01:54,387 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 51b4277372e700eb3a1230f9ab87509e in 1281ms, sequenceid=421, compaction requested=false 2024-12-06T16:01:54,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:54,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:54,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-06T16:01:54,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-06T16:01:54,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-06T16:01:54,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7430 sec 2024-12-06T16:01:54,394 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.7510 sec 2024-12-06T16:01:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:54,555 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:01:54,555 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 51b4277372e700eb3a1230f9ab87509e 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-12-06T16:01:54,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/a5e0bdb9e8dc4e87b1b0fa2906eba47c is 50, key is test_row_0/A:col10/1733500913403/Put/seqid=0 2024-12-06T16:01:54,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59456 deadline: 1733500974566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:54,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:01:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1733500974568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:01:54,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:54,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:01:54,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:01:54,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742179_1355 (size=14741) 2024-12-06T16:01:54,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/a5e0bdb9e8dc4e87b1b0fa2906eba47c 2024-12-06T16:01:54,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/034c6381a638418e8a8526cdec6185dc is 50, key is test_row_0/B:col10/1733500913403/Put/seqid=0 2024-12-06T16:01:54,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742180_1356 (size=12301) 2024-12-06T16:01:54,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/034c6381a638418e8a8526cdec6185dc 2024-12-06T16:01:54,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/c25254f12db6433f990b76daf1c0db1e is 50, key is test_row_0/C:col10/1733500913403/Put/seqid=0 2024-12-06T16:01:54,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742181_1357 (size=12301) 2024-12-06T16:01:54,643 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/c25254f12db6433f990b76daf1c0db1e 2024-12-06T16:01:54,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/a5e0bdb9e8dc4e87b1b0fa2906eba47c as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/a5e0bdb9e8dc4e87b1b0fa2906eba47c 2024-12-06T16:01:54,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/a5e0bdb9e8dc4e87b1b0fa2906eba47c, entries=200, sequenceid=451, filesize=14.4 K 2024-12-06T16:01:54,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/034c6381a638418e8a8526cdec6185dc as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/034c6381a638418e8a8526cdec6185dc 2024-12-06T16:01:54,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/034c6381a638418e8a8526cdec6185dc, entries=150, sequenceid=451, filesize=12.0 K 2024-12-06T16:01:54,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/c25254f12db6433f990b76daf1c0db1e as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/c25254f12db6433f990b76daf1c0db1e 2024-12-06T16:01:54,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/c25254f12db6433f990b76daf1c0db1e, entries=150, sequenceid=451, filesize=12.0 K 2024-12-06T16:01:54,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 51b4277372e700eb3a1230f9ab87509e in 117ms, sequenceid=451, compaction requested=true 2024-12-06T16:01:54,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:54,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51b4277372e700eb3a1230f9ab87509e:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:01:54,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:54,673 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:54,673 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:54,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51b4277372e700eb3a1230f9ab87509e:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:01:54,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:54,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51b4277372e700eb3a1230f9ab87509e:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:01:54,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:54,682 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:01:54,682 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 51b4277372e700eb3a1230f9ab87509e/A is initiating minor compaction (all files) 2024-12-06T16:01:54,682 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:01:54,682 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51b4277372e700eb3a1230f9ab87509e/A in TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:54,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 51b4277372e700eb3a1230f9ab87509e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-06T16:01:54,682 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/4d19516dd6654002b8af31bfed2f402c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/4afbedc7d0c145b88ec5f569339e2129, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/a5e0bdb9e8dc4e87b1b0fa2906eba47c] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp, totalSize=39.4 K 2024-12-06T16:01:54,682 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:54,682 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 51b4277372e700eb3a1230f9ab87509e/B is initiating minor compaction (all files) 2024-12-06T16:01:54,682 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51b4277372e700eb3a1230f9ab87509e/B in TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:54,682 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/606fe6ab655444d5871588cb5b992bc7, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/8ed407431cbc4d378d5aa3f731a47736, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/034c6381a638418e8a8526cdec6185dc] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp, totalSize=37.0 K 2024-12-06T16:01:54,684 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d19516dd6654002b8af31bfed2f402c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1733500911822 2024-12-06T16:01:54,684 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 606fe6ab655444d5871588cb5b992bc7, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1733500911822 2024-12-06T16:01:54,686 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ed407431cbc4d378d5aa3f731a47736, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1733500912163 2024-12-06T16:01:54,688 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 034c6381a638418e8a8526cdec6185dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1733500913403 2024-12-06T16:01:54,689 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4afbedc7d0c145b88ec5f569339e2129, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1733500912163 2024-12-06T16:01:54,690 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5e0bdb9e8dc4e87b1b0fa2906eba47c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1733500913403 2024-12-06T16:01:54,706 INFO [Thread-1230 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1230. 2024-12-06T16:01:54,706 INFO [Thread-1225 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1225. 2024-12-06T16:01:54,706 DEBUG [Thread-1225 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:01:54,706 DEBUG [Thread-1230 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:01:54,706 INFO [Thread-1232 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1232. 2024-12-06T16:01:54,706 DEBUG [Thread-1225 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:54,706 DEBUG [Thread-1230 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:54,706 DEBUG [Thread-1232 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:01:54,706 DEBUG [Thread-1232 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:54,706 DEBUG [Thread-1225 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:54,706 DEBUG [Thread-1232 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:54,706 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:01:54,706 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:01:54,707 DEBUG [Thread-1230 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:54,707 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:01:54,708 INFO [Thread-1234 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1234. 2024-12-06T16:01:54,708 DEBUG [Thread-1234 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:01:54,708 DEBUG [Thread-1234 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:54,708 DEBUG [Thread-1234 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:54,708 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:01:54,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/217374aab47e4eeaa1c4d04a1bf2df29 is 50, key is test_row_0/A:col10/1733500914680/Put/seqid=0 2024-12-06T16:01:54,709 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51b4277372e700eb3a1230f9ab87509e#B#compaction#310 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:54,710 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/6437f4f122f64785a3ebad7e95861fd1 is 50, key is test_row_0/B:col10/1733500913403/Put/seqid=0 2024-12-06T16:01:54,711 INFO [Thread-1219 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1219. 2024-12-06T16:01:54,711 DEBUG [Thread-1219 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:01:54,711 DEBUG [Thread-1219 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:54,711 DEBUG [Thread-1219 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:54,711 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:01:54,712 INFO [Thread-1228 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1228. 2024-12-06T16:01:54,712 DEBUG [Thread-1228 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:01:54,712 DEBUG [Thread-1228 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:54,712 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:01:54,713 DEBUG [Thread-1228 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:54,714 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51b4277372e700eb3a1230f9ab87509e#A#compaction#311 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:54,715 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/9046354537bc4762a38a19a11f66aee8 is 50, key is test_row_0/A:col10/1733500913403/Put/seqid=0 2024-12-06T16:01:54,715 INFO [Thread-1236 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1236. 2024-12-06T16:01:54,715 DEBUG [Thread-1236 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:01:54,715 DEBUG [Thread-1236 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:54,715 DEBUG [Thread-1236 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:01:54,715 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:01:54,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742182_1358 (size=12301) 2024-12-06T16:01:54,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/217374aab47e4eeaa1c4d04a1bf2df29 2024-12-06T16:01:54,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742183_1359 (size=13357) 2024-12-06T16:01:54,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/12995b9b64a1471ca04ff5ece21489c0 is 50, key is test_row_0/B:col10/1733500914680/Put/seqid=0 2024-12-06T16:01:54,739 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/6437f4f122f64785a3ebad7e95861fd1 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/6437f4f122f64785a3ebad7e95861fd1 2024-12-06T16:01:54,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742184_1360 (size=13357) 2024-12-06T16:01:54,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742185_1361 (size=12301) 2024-12-06T16:01:54,746 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51b4277372e700eb3a1230f9ab87509e/B of 51b4277372e700eb3a1230f9ab87509e into 6437f4f122f64785a3ebad7e95861fd1(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:54,747 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:54,747 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., storeName=51b4277372e700eb3a1230f9ab87509e/B, priority=13, startTime=1733500914673; duration=0sec 2024-12-06T16:01:54,747 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:01:54,747 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51b4277372e700eb3a1230f9ab87509e:B 2024-12-06T16:01:54,747 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:01:54,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/12995b9b64a1471ca04ff5ece21489c0 2024-12-06T16:01:54,748 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:01:54,749 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 51b4277372e700eb3a1230f9ab87509e/C is initiating minor compaction (all files) 2024-12-06T16:01:54,749 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 51b4277372e700eb3a1230f9ab87509e/C in TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:01:54,749 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/0f78ac425648446e903593c9609a3411, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/3dcabfe4411e4a4387ef69881ceddc9a, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/c25254f12db6433f990b76daf1c0db1e] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp, totalSize=37.0 K 2024-12-06T16:01:54,749 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f78ac425648446e903593c9609a3411, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1733500911822 2024-12-06T16:01:54,749 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dcabfe4411e4a4387ef69881ceddc9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1733500912163 2024-12-06T16:01:54,750 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting c25254f12db6433f990b76daf1c0db1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1733500913403 2024-12-06T16:01:54,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/150104ebda354a8387464f5013814ad5 is 50, key is test_row_0/C:col10/1733500914680/Put/seqid=0 2024-12-06T16:01:54,756 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 51b4277372e700eb3a1230f9ab87509e#C#compaction#314 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:01:54,757 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/60e53061509a4c7f97fb3e98f49eab59 is 50, key is test_row_0/C:col10/1733500913403/Put/seqid=0 2024-12-06T16:01:54,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742186_1362 (size=12301) 2024-12-06T16:01:54,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742187_1363 (size=13357) 2024-12-06T16:01:54,766 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/60e53061509a4c7f97fb3e98f49eab59 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/60e53061509a4c7f97fb3e98f49eab59 2024-12-06T16:01:54,771 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51b4277372e700eb3a1230f9ab87509e/C of 51b4277372e700eb3a1230f9ab87509e into 60e53061509a4c7f97fb3e98f49eab59(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:54,771 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:54,771 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., storeName=51b4277372e700eb3a1230f9ab87509e/C, priority=13, startTime=1733500914673; duration=0sec 2024-12-06T16:01:54,771 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:54,771 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51b4277372e700eb3a1230f9ab87509e:C 2024-12-06T16:01:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-06T16:01:54,792 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:01:55,150 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/9046354537bc4762a38a19a11f66aee8 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/9046354537bc4762a38a19a11f66aee8 2024-12-06T16:01:55,155 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 51b4277372e700eb3a1230f9ab87509e/A of 51b4277372e700eb3a1230f9ab87509e into 9046354537bc4762a38a19a11f66aee8(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:01:55,155 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:55,156 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., storeName=51b4277372e700eb3a1230f9ab87509e/A, priority=13, startTime=1733500914673; duration=0sec 2024-12-06T16:01:55,156 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:01:55,156 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51b4277372e700eb3a1230f9ab87509e:A 2024-12-06T16:01:55,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/150104ebda354a8387464f5013814ad5 2024-12-06T16:01:55,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/217374aab47e4eeaa1c4d04a1bf2df29 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/217374aab47e4eeaa1c4d04a1bf2df29 2024-12-06T16:01:55,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/217374aab47e4eeaa1c4d04a1bf2df29, entries=150, sequenceid=462, filesize=12.0 K 2024-12-06T16:01:55,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/12995b9b64a1471ca04ff5ece21489c0 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/12995b9b64a1471ca04ff5ece21489c0 2024-12-06T16:01:55,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/12995b9b64a1471ca04ff5ece21489c0, entries=150, sequenceid=462, filesize=12.0 K 2024-12-06T16:01:55,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/150104ebda354a8387464f5013814ad5 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/150104ebda354a8387464f5013814ad5 2024-12-06T16:01:55,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/150104ebda354a8387464f5013814ad5, entries=150, sequenceid=462, filesize=12.0 K 2024-12-06T16:01:55,183 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=60.38 KB/61830 for 51b4277372e700eb3a1230f9ab87509e in 501ms, sequenceid=462, compaction requested=false 2024-12-06T16:01:55,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:01:55,827 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-06T16:02:01,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:02:01,574 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:02:01,574 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 51b4277372e700eb3a1230f9ab87509e 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-06T16:02:01,574 INFO [Thread-1221 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1221. 2024-12-06T16:02:01,574 DEBUG [Thread-1221 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:02:01,574 DEBUG [Thread-1221 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:01,574 DEBUG [Thread-1221 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:01,574 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:02:01,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/99346a9cd95149b3b072bb883a444ce7 is 50, key is test_row_0/A:col10/1733500921572/Put/seqid=0 2024-12-06T16:02:01,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742188_1364 (size=12301) 2024-12-06T16:02:01,582 INFO [Thread-1217 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1217. 2024-12-06T16:02:01,583 DEBUG [Thread-1217 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:02:01,583 DEBUG [Thread-1217 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:01,583 DEBUG [Thread-1217 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:01,583 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:02:01,603 INFO [Thread-1223 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1223. 2024-12-06T16:02:01,603 DEBUG [Thread-1223 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:02:01,603 DEBUG [Thread-1223 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:01,603 DEBUG [Thread-1223 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:01,603 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:02:01,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-06T16:02:01,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-12-06T16:02:01,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 118 2024-12-06T16:02:01,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-12-06T16:02:01,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-12-06T16:02:01,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 132 2024-12-06T16:02:01,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-06T16:02:01,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2900 2024-12-06T16:02:01,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2990 2024-12-06T16:02:01,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2806 2024-12-06T16:02:01,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2911 2024-12-06T16:02:01,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2923 2024-12-06T16:02:01,603 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-06T16:02:01,604 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T16:02:01,604 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T16:02:01,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:01,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:01,604 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:02:01,604 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-06T16:02:01,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-06T16:02:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-06T16:02:01,607 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733500921607"}]},"ts":"1733500921607"} 2024-12-06T16:02:01,609 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-06T16:02:01,609 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-06T16:02:01,609 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-06T16:02:01,610 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=51b4277372e700eb3a1230f9ab87509e, UNASSIGN}] 2024-12-06T16:02:01,611 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=51b4277372e700eb3a1230f9ab87509e, UNASSIGN 2024-12-06T16:02:01,612 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=51b4277372e700eb3a1230f9ab87509e, regionState=CLOSING, regionLocation=85bef17d9292,46105,1733500827732 2024-12-06T16:02:01,613 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=51b4277372e700eb3a1230f9ab87509e, UNASSIGN because future has completed 2024-12-06T16:02:01,613 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-06T16:02:01,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; CloseRegionProcedure 51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732}] 2024-12-06T16:02:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-06T16:02:01,765 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(122): Close 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:02:01,765 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-06T16:02:01,765 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1722): Closing 51b4277372e700eb3a1230f9ab87509e, disabling compactions & flushes 2024-12-06T16:02:01,765 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:02:01,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-06T16:02:01,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/99346a9cd95149b3b072bb883a444ce7 2024-12-06T16:02:01,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/6574a2cb51364e5da97d339f1546aca1 is 50, key is test_row_0/B:col10/1733500921572/Put/seqid=0 2024-12-06T16:02:01,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742189_1365 (size=12301) 2024-12-06T16:02:02,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-06T16:02:02,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/6574a2cb51364e5da97d339f1546aca1 2024-12-06T16:02:02,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/45e1e891f2b74e9787be8f51d5514a44 is 50, key is test_row_0/C:col10/1733500921572/Put/seqid=0 2024-12-06T16:02:02,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742190_1366 (size=12301) 2024-12-06T16:02:02,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-06T16:02:02,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/45e1e891f2b74e9787be8f51d5514a44 2024-12-06T16:02:02,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/99346a9cd95149b3b072bb883a444ce7 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/99346a9cd95149b3b072bb883a444ce7 2024-12-06T16:02:02,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/99346a9cd95149b3b072bb883a444ce7, entries=150, sequenceid=478, filesize=12.0 K 2024-12-06T16:02:02,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/6574a2cb51364e5da97d339f1546aca1 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/6574a2cb51364e5da97d339f1546aca1 2024-12-06T16:02:02,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/6574a2cb51364e5da97d339f1546aca1, entries=150, sequenceid=478, filesize=12.0 K 2024-12-06T16:02:02,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/45e1e891f2b74e9787be8f51d5514a44 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/45e1e891f2b74e9787be8f51d5514a44 2024-12-06T16:02:02,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/45e1e891f2b74e9787be8f51d5514a44, entries=150, sequenceid=478, filesize=12.0 K 2024-12-06T16:02:02,817 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=13.42 KB/13740 for 51b4277372e700eb3a1230f9ab87509e in 1243ms, sequenceid=478, compaction requested=true 2024-12-06T16:02:02,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 51b4277372e700eb3a1230f9ab87509e: 2024-12-06T16:02:02,817 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:02:02,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51b4277372e700eb3a1230f9ab87509e:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:02:02,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:02,817 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:02:02,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51b4277372e700eb3a1230f9ab87509e:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:02:02,817 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. after waiting 0 ms 2024-12-06T16:02:02,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:02,817 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:02:02,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 51b4277372e700eb3a1230f9ab87509e:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:02:02,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:02,818 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(2902): Flushing 51b4277372e700eb3a1230f9ab87509e 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-12-06T16:02:02,818 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. because compaction request was cancelled 2024-12-06T16:02:02,818 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. because compaction request was cancelled 2024-12-06T16:02:02,818 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51b4277372e700eb3a1230f9ab87509e:B 2024-12-06T16:02:02,818 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51b4277372e700eb3a1230f9ab87509e:A 2024-12-06T16:02:02,818 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. because compaction request was cancelled 2024-12-06T16:02:02,818 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 51b4277372e700eb3a1230f9ab87509e:C 2024-12-06T16:02:02,821 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/82e6cc27f8f44135b69042617794bdd1 is 50, key is test_row_1/A:col10/1733500921602/Put/seqid=0 2024-12-06T16:02:02,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742191_1367 (size=7415) 2024-12-06T16:02:03,224 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/82e6cc27f8f44135b69042617794bdd1 2024-12-06T16:02:03,231 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/f31c65ad69e744e3be9f8c173aa7e33b is 50, key is test_row_1/B:col10/1733500921602/Put/seqid=0 2024-12-06T16:02:03,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742192_1368 (size=7415) 2024-12-06T16:02:03,635 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/f31c65ad69e744e3be9f8c173aa7e33b 2024-12-06T16:02:03,641 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/f1def4e71fdf47ff935a71d98e8332ba is 50, key is test_row_1/C:col10/1733500921602/Put/seqid=0 2024-12-06T16:02:03,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742193_1369 (size=7415) 2024-12-06T16:02:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-06T16:02:04,044 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/f1def4e71fdf47ff935a71d98e8332ba 2024-12-06T16:02:04,048 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/A/82e6cc27f8f44135b69042617794bdd1 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/82e6cc27f8f44135b69042617794bdd1 2024-12-06T16:02:04,051 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/82e6cc27f8f44135b69042617794bdd1, entries=50, sequenceid=483, filesize=7.2 K 2024-12-06T16:02:04,052 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/B/f31c65ad69e744e3be9f8c173aa7e33b as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/f31c65ad69e744e3be9f8c173aa7e33b 2024-12-06T16:02:04,055 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/f31c65ad69e744e3be9f8c173aa7e33b, entries=50, sequenceid=483, filesize=7.2 K 2024-12-06T16:02:04,056 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/.tmp/C/f1def4e71fdf47ff935a71d98e8332ba as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/f1def4e71fdf47ff935a71d98e8332ba 2024-12-06T16:02:04,059 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/f1def4e71fdf47ff935a71d98e8332ba, entries=50, sequenceid=483, filesize=7.2 K 2024-12-06T16:02:04,059 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 51b4277372e700eb3a1230f9ab87509e in 1241ms, sequenceid=483, compaction requested=true 2024-12-06T16:02:04,060 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/d4769b9638f7432c89cb877f718e5f00, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/04a4c35c74ad4df8a99c0ce8c23ac03b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/a76e1bc1c05d473db38f4463b07bc397, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/c5166b912a0b42e48726d05d12bbed36, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/e6bf054385c74a03894b1ddeefe6938f, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/446fe5bb56de445688f004e696d64274, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/0ab02b9940a24ded910174bddb18018f, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/02dbe71a7c7e4362a09db13b7153652b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/f3e2f540b5da494dbd3342a1d7ecafaa, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/e03471c527024cb3bc33c118bd25e63c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/6c87bebfa4e941cab80c0df69b629ea1, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/b5c2fc7bb9154bfeafa60aac0d62f0b5, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/7608eadefda84735afbc320627b633da, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/5655ea36bbaa4c5fac501b1da6ea79fc, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/443fa7d4b967468c89c2b85adca88372, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/c17864e13b1f4e40be56e4596bf14831, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/53e7db616c1b4865a364f5eceecff382, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/9d0adce57946467ba8dd744f2ab61b58, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/82b1a32e011a459e8cb4170b596bb838, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/d4261daea5474d2596ed3cbeffd1ae97, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/e165deddab9944a8b40e2107094f4ac0, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/ffd0c99559904f2b9bdc92678a31e008, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/d40cb36c901e4663b0252b63ac5a524b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/6ee96f02ab2f49eda64291d6d97f4d09, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/d16d883584c144f5bb43c636a4f27a30, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/32dbacf8ce8b4cb68fe950978d54872d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/49bbc8a22c924bdf9c14d17f093b6726, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/b2d796541d344ef7aee886fe2e52f894, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/4d19516dd6654002b8af31bfed2f402c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/4afbedc7d0c145b88ec5f569339e2129, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/a5e0bdb9e8dc4e87b1b0fa2906eba47c] to archive 2024-12-06T16:02:04,061 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T16:02:04,062 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/d4769b9638f7432c89cb877f718e5f00 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/d4769b9638f7432c89cb877f718e5f00 2024-12-06T16:02:04,063 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/04a4c35c74ad4df8a99c0ce8c23ac03b to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/04a4c35c74ad4df8a99c0ce8c23ac03b 2024-12-06T16:02:04,064 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/a76e1bc1c05d473db38f4463b07bc397 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/a76e1bc1c05d473db38f4463b07bc397 2024-12-06T16:02:04,065 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/c5166b912a0b42e48726d05d12bbed36 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/c5166b912a0b42e48726d05d12bbed36 2024-12-06T16:02:04,066 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/e6bf054385c74a03894b1ddeefe6938f to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/e6bf054385c74a03894b1ddeefe6938f 2024-12-06T16:02:04,067 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/446fe5bb56de445688f004e696d64274 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/446fe5bb56de445688f004e696d64274 2024-12-06T16:02:04,068 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/0ab02b9940a24ded910174bddb18018f to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/0ab02b9940a24ded910174bddb18018f 2024-12-06T16:02:04,069 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/02dbe71a7c7e4362a09db13b7153652b to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/02dbe71a7c7e4362a09db13b7153652b 2024-12-06T16:02:04,070 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/f3e2f540b5da494dbd3342a1d7ecafaa to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/f3e2f540b5da494dbd3342a1d7ecafaa 2024-12-06T16:02:04,071 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/e03471c527024cb3bc33c118bd25e63c to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/e03471c527024cb3bc33c118bd25e63c 2024-12-06T16:02:04,071 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/6c87bebfa4e941cab80c0df69b629ea1 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/6c87bebfa4e941cab80c0df69b629ea1 2024-12-06T16:02:04,072 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/b5c2fc7bb9154bfeafa60aac0d62f0b5 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/b5c2fc7bb9154bfeafa60aac0d62f0b5 2024-12-06T16:02:04,073 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/7608eadefda84735afbc320627b633da to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/7608eadefda84735afbc320627b633da 2024-12-06T16:02:04,074 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/5655ea36bbaa4c5fac501b1da6ea79fc to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/5655ea36bbaa4c5fac501b1da6ea79fc 2024-12-06T16:02:04,075 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/443fa7d4b967468c89c2b85adca88372 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/443fa7d4b967468c89c2b85adca88372 2024-12-06T16:02:04,076 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/c17864e13b1f4e40be56e4596bf14831 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/c17864e13b1f4e40be56e4596bf14831 2024-12-06T16:02:04,077 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/53e7db616c1b4865a364f5eceecff382 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/53e7db616c1b4865a364f5eceecff382 2024-12-06T16:02:04,078 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/9d0adce57946467ba8dd744f2ab61b58 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/9d0adce57946467ba8dd744f2ab61b58 2024-12-06T16:02:04,079 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/82b1a32e011a459e8cb4170b596bb838 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/82b1a32e011a459e8cb4170b596bb838 2024-12-06T16:02:04,080 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/d4261daea5474d2596ed3cbeffd1ae97 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/d4261daea5474d2596ed3cbeffd1ae97 2024-12-06T16:02:04,081 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/e165deddab9944a8b40e2107094f4ac0 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/e165deddab9944a8b40e2107094f4ac0 2024-12-06T16:02:04,082 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/ffd0c99559904f2b9bdc92678a31e008 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/ffd0c99559904f2b9bdc92678a31e008 2024-12-06T16:02:04,083 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/d40cb36c901e4663b0252b63ac5a524b to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/d40cb36c901e4663b0252b63ac5a524b 2024-12-06T16:02:04,083 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/6ee96f02ab2f49eda64291d6d97f4d09 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/6ee96f02ab2f49eda64291d6d97f4d09 2024-12-06T16:02:04,084 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/d16d883584c144f5bb43c636a4f27a30 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/d16d883584c144f5bb43c636a4f27a30 2024-12-06T16:02:04,085 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/32dbacf8ce8b4cb68fe950978d54872d to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/32dbacf8ce8b4cb68fe950978d54872d 2024-12-06T16:02:04,086 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/49bbc8a22c924bdf9c14d17f093b6726 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/49bbc8a22c924bdf9c14d17f093b6726 2024-12-06T16:02:04,087 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/b2d796541d344ef7aee886fe2e52f894 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/b2d796541d344ef7aee886fe2e52f894 2024-12-06T16:02:04,088 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/4d19516dd6654002b8af31bfed2f402c to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/4d19516dd6654002b8af31bfed2f402c 2024-12-06T16:02:04,088 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/4afbedc7d0c145b88ec5f569339e2129 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/4afbedc7d0c145b88ec5f569339e2129 2024-12-06T16:02:04,089 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/a5e0bdb9e8dc4e87b1b0fa2906eba47c to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/a5e0bdb9e8dc4e87b1b0fa2906eba47c 2024-12-06T16:02:04,091 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/27d16d397cb646809e6703427cd44209, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/c6abaf7986ce4655809425d17ce64e56, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/147c6e72cea54d569c8598e8e7922fb9, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/7869b0138be74fafa661b787d5ec9239, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/ed176bccfb494baba0490be5fe13fc0a, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/766aea599d9d4534949235e9daa7f3ca, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/5f12385f3e464cf4aafdf98ffe60c61e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/3e3096822aab4f3aaf10f5b117673812, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/ad25b6143a9b4b9ea5fd7ba3952ebf1c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/4ecf9ab0dcf44e278dc91fe548583931, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/4e174896b8d240c9bb9155141ee15fb9, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/51cf2f94795d4233b6a79aac8620d2d2, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/9cff1684d47a410f93ae4a3fe4b571a0, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/5214392df5b74a87a6a35745ae92f1bf, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/f2828fc5e54d41ac8dacf11ab1714046, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/8dcb15b81eee43f2ba75a29fca744c7b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/1247cb9b83154f24bdbf7bde819eb121, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/2d98b9dc2e7b4388a9d295d9191d2efe, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/0c3fa04f14f64c48aa11582d87020ac9, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/d01ffd61d40d4555b397ccf570a3a719, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/8158b0edd1094777a345285a53671c65, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/4096f9cab52e4466928561cd3c539b3a, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/96ff259437964f06a6a558d8631fbb6b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/ebb2c2a24eb14fb8b0d9c46835df6e1e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/a969cac3bc194692b996f2f57769e777, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/f02941321e03440b970979db92ee39d3, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/c7ae5eb9b5bb4828bf688307031e2154, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/606fe6ab655444d5871588cb5b992bc7, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/70456bed0aa1496ab4166fafb6d25626, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/8ed407431cbc4d378d5aa3f731a47736, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/034c6381a638418e8a8526cdec6185dc] to archive 2024-12-06T16:02:04,092 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T16:02:04,094 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/27d16d397cb646809e6703427cd44209 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/27d16d397cb646809e6703427cd44209 2024-12-06T16:02:04,095 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/c6abaf7986ce4655809425d17ce64e56 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/c6abaf7986ce4655809425d17ce64e56 2024-12-06T16:02:04,095 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/147c6e72cea54d569c8598e8e7922fb9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/147c6e72cea54d569c8598e8e7922fb9 2024-12-06T16:02:04,096 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/7869b0138be74fafa661b787d5ec9239 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/7869b0138be74fafa661b787d5ec9239 2024-12-06T16:02:04,097 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/ed176bccfb494baba0490be5fe13fc0a to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/ed176bccfb494baba0490be5fe13fc0a 2024-12-06T16:02:04,098 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/766aea599d9d4534949235e9daa7f3ca to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/766aea599d9d4534949235e9daa7f3ca 2024-12-06T16:02:04,099 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/5f12385f3e464cf4aafdf98ffe60c61e to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/5f12385f3e464cf4aafdf98ffe60c61e 2024-12-06T16:02:04,100 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/3e3096822aab4f3aaf10f5b117673812 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/3e3096822aab4f3aaf10f5b117673812 2024-12-06T16:02:04,101 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/ad25b6143a9b4b9ea5fd7ba3952ebf1c to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/ad25b6143a9b4b9ea5fd7ba3952ebf1c 2024-12-06T16:02:04,102 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/4ecf9ab0dcf44e278dc91fe548583931 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/4ecf9ab0dcf44e278dc91fe548583931 2024-12-06T16:02:04,103 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/4e174896b8d240c9bb9155141ee15fb9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/4e174896b8d240c9bb9155141ee15fb9 2024-12-06T16:02:04,104 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/51cf2f94795d4233b6a79aac8620d2d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/51cf2f94795d4233b6a79aac8620d2d2 2024-12-06T16:02:04,105 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/9cff1684d47a410f93ae4a3fe4b571a0 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/9cff1684d47a410f93ae4a3fe4b571a0 2024-12-06T16:02:04,106 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/5214392df5b74a87a6a35745ae92f1bf to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/5214392df5b74a87a6a35745ae92f1bf 2024-12-06T16:02:04,107 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/f2828fc5e54d41ac8dacf11ab1714046 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/f2828fc5e54d41ac8dacf11ab1714046 2024-12-06T16:02:04,108 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/8dcb15b81eee43f2ba75a29fca744c7b to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/8dcb15b81eee43f2ba75a29fca744c7b 2024-12-06T16:02:04,109 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/1247cb9b83154f24bdbf7bde819eb121 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/1247cb9b83154f24bdbf7bde819eb121 2024-12-06T16:02:04,110 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/2d98b9dc2e7b4388a9d295d9191d2efe to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/2d98b9dc2e7b4388a9d295d9191d2efe 2024-12-06T16:02:04,111 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/0c3fa04f14f64c48aa11582d87020ac9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/0c3fa04f14f64c48aa11582d87020ac9 2024-12-06T16:02:04,112 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/d01ffd61d40d4555b397ccf570a3a719 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/d01ffd61d40d4555b397ccf570a3a719 2024-12-06T16:02:04,113 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/8158b0edd1094777a345285a53671c65 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/8158b0edd1094777a345285a53671c65 2024-12-06T16:02:04,114 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/4096f9cab52e4466928561cd3c539b3a to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/4096f9cab52e4466928561cd3c539b3a 2024-12-06T16:02:04,115 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/96ff259437964f06a6a558d8631fbb6b to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/96ff259437964f06a6a558d8631fbb6b 2024-12-06T16:02:04,116 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/ebb2c2a24eb14fb8b0d9c46835df6e1e to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/ebb2c2a24eb14fb8b0d9c46835df6e1e 2024-12-06T16:02:04,118 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/a969cac3bc194692b996f2f57769e777 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/a969cac3bc194692b996f2f57769e777 2024-12-06T16:02:04,119 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/f02941321e03440b970979db92ee39d3 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/f02941321e03440b970979db92ee39d3 2024-12-06T16:02:04,120 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/c7ae5eb9b5bb4828bf688307031e2154 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/c7ae5eb9b5bb4828bf688307031e2154 2024-12-06T16:02:04,122 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/606fe6ab655444d5871588cb5b992bc7 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/606fe6ab655444d5871588cb5b992bc7 2024-12-06T16:02:04,123 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/70456bed0aa1496ab4166fafb6d25626 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/70456bed0aa1496ab4166fafb6d25626 2024-12-06T16:02:04,124 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/8ed407431cbc4d378d5aa3f731a47736 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/8ed407431cbc4d378d5aa3f731a47736 2024-12-06T16:02:04,125 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/034c6381a638418e8a8526cdec6185dc to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/034c6381a638418e8a8526cdec6185dc 2024-12-06T16:02:04,127 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/2e12751d30b746289abc3489e30fa292, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/a59d9be6e0584c8ea32fc8325ac954fe, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/47f96edccaa6479898962a74670fb262, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/90a3c06c1fb94f56827fcdf2f03bbcf5, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/bc1f62efb40d4c4a9399f9cc16b207dc, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/5c2f7341034f4ec8bc28c19d090bb37c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/710ceb21e8844739a161a19f2f20fe24, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/817ce68722fd4d5cb2ea19906156bc52, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/e797936106bb40a09e17849d69b0850d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/851c9d5a5b704de8aaf3c3445c28e3d1, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/985fce4699bd412a88572535386c7c09, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/37d26045da874c629ae099ddf2c856b1, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/dd56a78d708643e6892a679ecd3d1aeb, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/5e34ca72e1b1497b88bbb28cf0aa9e21, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/0b437ec3634c4824af7ca3f09a87227b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/1129af84a991422981fc3e5f2024913a, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/3262d57c1b484679bf248a3c269c6589, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/6df4f1bf5ec241fe8cb95c6e8bf6bcc0, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/4bd45bdbe5674c6db7ce47792463d7ad, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/0b7e6c5a089f4e7492e1275df2ecd290, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/e8836b40b79e4d6bb7a39304562d1ba8, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/20f51eb663f54fe28ce8ccb411858eab, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/04c172ad2b3248fd983c87e3abc88a08, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/77e64d3a33be4222bc60e9ab15f46fd7, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/32063ce73792434db3ee4a2a4ff6aee3, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/e93afe066202466bb744da22b460390c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/fb05d8982dc14f71bac1f1af391616db, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/0f78ac425648446e903593c9609a3411, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/1dee31450b7a4e7e99fb7d7d83def59c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/3dcabfe4411e4a4387ef69881ceddc9a, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/c25254f12db6433f990b76daf1c0db1e] to archive 2024-12-06T16:02:04,128 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T16:02:04,130 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/2e12751d30b746289abc3489e30fa292 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/2e12751d30b746289abc3489e30fa292 2024-12-06T16:02:04,131 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/a59d9be6e0584c8ea32fc8325ac954fe to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/a59d9be6e0584c8ea32fc8325ac954fe 2024-12-06T16:02:04,133 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/47f96edccaa6479898962a74670fb262 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/47f96edccaa6479898962a74670fb262 2024-12-06T16:02:04,134 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/90a3c06c1fb94f56827fcdf2f03bbcf5 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/90a3c06c1fb94f56827fcdf2f03bbcf5 2024-12-06T16:02:04,135 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/bc1f62efb40d4c4a9399f9cc16b207dc to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/bc1f62efb40d4c4a9399f9cc16b207dc 2024-12-06T16:02:04,136 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/5c2f7341034f4ec8bc28c19d090bb37c to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/5c2f7341034f4ec8bc28c19d090bb37c 2024-12-06T16:02:04,137 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/710ceb21e8844739a161a19f2f20fe24 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/710ceb21e8844739a161a19f2f20fe24 2024-12-06T16:02:04,138 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/817ce68722fd4d5cb2ea19906156bc52 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/817ce68722fd4d5cb2ea19906156bc52 2024-12-06T16:02:04,139 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/e797936106bb40a09e17849d69b0850d to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/e797936106bb40a09e17849d69b0850d 2024-12-06T16:02:04,140 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/851c9d5a5b704de8aaf3c3445c28e3d1 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/851c9d5a5b704de8aaf3c3445c28e3d1 2024-12-06T16:02:04,141 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/985fce4699bd412a88572535386c7c09 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/985fce4699bd412a88572535386c7c09 2024-12-06T16:02:04,142 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/37d26045da874c629ae099ddf2c856b1 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/37d26045da874c629ae099ddf2c856b1 2024-12-06T16:02:04,143 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/dd56a78d708643e6892a679ecd3d1aeb to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/dd56a78d708643e6892a679ecd3d1aeb 2024-12-06T16:02:04,144 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/5e34ca72e1b1497b88bbb28cf0aa9e21 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/5e34ca72e1b1497b88bbb28cf0aa9e21 2024-12-06T16:02:04,145 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/0b437ec3634c4824af7ca3f09a87227b to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/0b437ec3634c4824af7ca3f09a87227b 2024-12-06T16:02:04,145 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/1129af84a991422981fc3e5f2024913a to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/1129af84a991422981fc3e5f2024913a 2024-12-06T16:02:04,146 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/3262d57c1b484679bf248a3c269c6589 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/3262d57c1b484679bf248a3c269c6589 2024-12-06T16:02:04,147 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/6df4f1bf5ec241fe8cb95c6e8bf6bcc0 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/6df4f1bf5ec241fe8cb95c6e8bf6bcc0 2024-12-06T16:02:04,148 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/4bd45bdbe5674c6db7ce47792463d7ad to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/4bd45bdbe5674c6db7ce47792463d7ad 2024-12-06T16:02:04,149 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/0b7e6c5a089f4e7492e1275df2ecd290 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/0b7e6c5a089f4e7492e1275df2ecd290 2024-12-06T16:02:04,149 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/e8836b40b79e4d6bb7a39304562d1ba8 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/e8836b40b79e4d6bb7a39304562d1ba8 2024-12-06T16:02:04,150 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/20f51eb663f54fe28ce8ccb411858eab to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/20f51eb663f54fe28ce8ccb411858eab 2024-12-06T16:02:04,151 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/04c172ad2b3248fd983c87e3abc88a08 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/04c172ad2b3248fd983c87e3abc88a08 2024-12-06T16:02:04,152 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/77e64d3a33be4222bc60e9ab15f46fd7 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/77e64d3a33be4222bc60e9ab15f46fd7 2024-12-06T16:02:04,153 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/32063ce73792434db3ee4a2a4ff6aee3 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/32063ce73792434db3ee4a2a4ff6aee3 2024-12-06T16:02:04,153 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/e93afe066202466bb744da22b460390c to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/e93afe066202466bb744da22b460390c 2024-12-06T16:02:04,154 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/fb05d8982dc14f71bac1f1af391616db to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/fb05d8982dc14f71bac1f1af391616db 2024-12-06T16:02:04,155 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/0f78ac425648446e903593c9609a3411 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/0f78ac425648446e903593c9609a3411 2024-12-06T16:02:04,156 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/1dee31450b7a4e7e99fb7d7d83def59c to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/1dee31450b7a4e7e99fb7d7d83def59c 2024-12-06T16:02:04,157 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/3dcabfe4411e4a4387ef69881ceddc9a to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/3dcabfe4411e4a4387ef69881ceddc9a 2024-12-06T16:02:04,158 DEBUG [StoreCloser-TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/c25254f12db6433f990b76daf1c0db1e to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/c25254f12db6433f990b76daf1c0db1e 2024-12-06T16:02:04,162 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/recovered.edits/486.seqid, newMaxSeqId=486, maxSeqId=1 2024-12-06T16:02:04,162 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. 2024-12-06T16:02:04,162 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1676): Region close journal for 51b4277372e700eb3a1230f9ab87509e: Waiting for close lock at 1733500921765Running coprocessor pre-close hooks at 1733500921765Disabling compacts and flushes for region at 1733500921765Disabling writes for close at 1733500922817 (+1052 ms)Obtaining lock to block concurrent updates at 1733500922818 (+1 ms)Preparing flush snapshotting stores in 51b4277372e700eb3a1230f9ab87509e at 1733500922818Finished memstore snapshotting TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1733500922818Flushing stores of TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e. at 1733500922818Flushing 51b4277372e700eb3a1230f9ab87509e/A: creating writer at 1733500922818Flushing 51b4277372e700eb3a1230f9ab87509e/A: appending metadata at 1733500922820 (+2 ms)Flushing 51b4277372e700eb3a1230f9ab87509e/A: closing flushed file at 1733500922820Flushing 51b4277372e700eb3a1230f9ab87509e/B: creating writer at 1733500923228 (+408 ms)Flushing 51b4277372e700eb3a1230f9ab87509e/B: appending metadata at 1733500923230 (+2 ms)Flushing 51b4277372e700eb3a1230f9ab87509e/B: closing flushed file at 1733500923230Flushing 51b4277372e700eb3a1230f9ab87509e/C: creating writer at 1733500923638 (+408 ms)Flushing 51b4277372e700eb3a1230f9ab87509e/C: appending metadata at 1733500923640 (+2 ms)Flushing 51b4277372e700eb3a1230f9ab87509e/C: closing flushed file at 1733500923640Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73b15b1e: reopening flushed file at 1733500924048 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@133b426f: reopening flushed file at 1733500924051 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37f4d39: reopening flushed file at 1733500924055 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 51b4277372e700eb3a1230f9ab87509e in 1241ms, sequenceid=483, compaction requested=true at 1733500924059 (+4 ms)Writing region close event to WAL at 1733500924159 (+100 ms)Running coprocessor post-close hooks at 1733500924162 (+3 ms)Closed at 1733500924162 2024-12-06T16:02:04,164 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(157): Closed 51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:02:04,165 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=51b4277372e700eb3a1230f9ab87509e, regionState=CLOSED 2024-12-06T16:02:04,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=75, ppid=74, state=RUNNABLE, hasLock=false; CloseRegionProcedure 51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 because future has completed 2024-12-06T16:02:04,168 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-06T16:02:04,168 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; CloseRegionProcedure 51b4277372e700eb3a1230f9ab87509e, server=85bef17d9292,46105,1733500827732 in 2.5530 sec 2024-12-06T16:02:04,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=74, resume processing ppid=73 2024-12-06T16:02:04,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, ppid=73, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=51b4277372e700eb3a1230f9ab87509e, UNASSIGN in 2.5580 sec 2024-12-06T16:02:04,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-06T16:02:04,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.5610 sec 2024-12-06T16:02:04,173 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733500924172"}]},"ts":"1733500924172"} 2024-12-06T16:02:04,174 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-06T16:02:04,174 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-06T16:02:04,175 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.5690 sec 2024-12-06T16:02:05,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-06T16:02:05,762 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:02:05,762 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-06T16:02:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:02:05,764 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=76, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:02:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-06T16:02:05,765 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=76, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:02:05,767 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:02:05,769 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A, FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B, FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C, FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/recovered.edits] 2024-12-06T16:02:05,771 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/217374aab47e4eeaa1c4d04a1bf2df29 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/217374aab47e4eeaa1c4d04a1bf2df29 2024-12-06T16:02:05,772 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/82e6cc27f8f44135b69042617794bdd1 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/82e6cc27f8f44135b69042617794bdd1 2024-12-06T16:02:05,773 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/9046354537bc4762a38a19a11f66aee8 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/9046354537bc4762a38a19a11f66aee8 2024-12-06T16:02:05,774 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/99346a9cd95149b3b072bb883a444ce7 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/A/99346a9cd95149b3b072bb883a444ce7 2024-12-06T16:02:05,776 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/12995b9b64a1471ca04ff5ece21489c0 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/12995b9b64a1471ca04ff5ece21489c0 2024-12-06T16:02:05,776 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/6437f4f122f64785a3ebad7e95861fd1 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/6437f4f122f64785a3ebad7e95861fd1 2024-12-06T16:02:05,777 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/6574a2cb51364e5da97d339f1546aca1 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/6574a2cb51364e5da97d339f1546aca1 2024-12-06T16:02:05,778 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/f31c65ad69e744e3be9f8c173aa7e33b to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/B/f31c65ad69e744e3be9f8c173aa7e33b 2024-12-06T16:02:05,780 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/150104ebda354a8387464f5013814ad5 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/150104ebda354a8387464f5013814ad5 2024-12-06T16:02:05,781 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/45e1e891f2b74e9787be8f51d5514a44 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/45e1e891f2b74e9787be8f51d5514a44 2024-12-06T16:02:05,782 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/60e53061509a4c7f97fb3e98f49eab59 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/60e53061509a4c7f97fb3e98f49eab59 2024-12-06T16:02:05,783 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/f1def4e71fdf47ff935a71d98e8332ba to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/C/f1def4e71fdf47ff935a71d98e8332ba 2024-12-06T16:02:05,785 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/recovered.edits/486.seqid to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e/recovered.edits/486.seqid 2024-12-06T16:02:05,785 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/51b4277372e700eb3a1230f9ab87509e 2024-12-06T16:02:05,785 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-06T16:02:05,787 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=76, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:02:05,789 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-06T16:02:05,791 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-06T16:02:05,792 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=76, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:02:05,792 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-06T16:02:05,792 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733500925792"}]},"ts":"9223372036854775807"} 2024-12-06T16:02:05,793 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-06T16:02:05,793 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 51b4277372e700eb3a1230f9ab87509e, NAME => 'TestAcidGuarantees,,1733500892486.51b4277372e700eb3a1230f9ab87509e.', STARTKEY => '', ENDKEY => ''}] 2024-12-06T16:02:05,793 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-06T16:02:05,794 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733500925793"}]},"ts":"9223372036854775807"} 2024-12-06T16:02:05,795 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-06T16:02:05,795 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=76, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:02:05,796 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 33 msec 2024-12-06T16:02:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-06T16:02:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-06T16:02:05,872 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:02:05,881 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238 (was 242), OpenFileDescriptor=445 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=519 (was 582), ProcessCount=11 (was 11), AvailableMemoryMB=8694 (was 8763) 2024-12-06T16:02:05,890 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=238, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=519, ProcessCount=11, AvailableMemoryMB=8694 2024-12-06T16:02:05,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-06T16:02:05,891 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T16:02:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=77, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-06T16:02:05,893 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T16:02:05,893 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:02:05,893 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 77 2024-12-06T16:02:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-12-06T16:02:05,894 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T16:02:05,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742194_1370 (size=959) 2024-12-06T16:02:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-12-06T16:02:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-12-06T16:02:06,300 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d 2024-12-06T16:02:06,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742195_1371 (size=53) 2024-12-06T16:02:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-12-06T16:02:06,706 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T16:02:06,706 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 81761a3d64f4403adedb17ba781753d2, disabling compactions & flushes 2024-12-06T16:02:06,706 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:06,706 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:06,706 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. after waiting 0 ms 2024-12-06T16:02:06,706 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:06,706 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:06,706 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 81761a3d64f4403adedb17ba781753d2: Waiting for close lock at 1733500926706Disabling compacts and flushes for region at 1733500926706Disabling writes for close at 1733500926706Writing region close event to WAL at 1733500926706Closed at 1733500926706 2024-12-06T16:02:06,708 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T16:02:06,708 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733500926708"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733500926708"}]},"ts":"1733500926708"} 2024-12-06T16:02:06,709 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T16:02:06,710 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T16:02:06,710 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733500926710"}]},"ts":"1733500926710"} 2024-12-06T16:02:06,712 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-06T16:02:06,712 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81761a3d64f4403adedb17ba781753d2, ASSIGN}] 2024-12-06T16:02:06,713 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81761a3d64f4403adedb17ba781753d2, ASSIGN 2024-12-06T16:02:06,713 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=81761a3d64f4403adedb17ba781753d2, ASSIGN; state=OFFLINE, location=85bef17d9292,46105,1733500827732; forceNewPlan=false, retain=false 2024-12-06T16:02:06,864 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=81761a3d64f4403adedb17ba781753d2, regionState=OPENING, regionLocation=85bef17d9292,46105,1733500827732 2024-12-06T16:02:06,866 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=81761a3d64f4403adedb17ba781753d2, ASSIGN because future has completed 2024-12-06T16:02:06,866 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732}] 2024-12-06T16:02:07,023 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:07,023 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} 2024-12-06T16:02:07,024 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:07,024 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T16:02:07,024 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:07,024 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:07,025 INFO [StoreOpener-81761a3d64f4403adedb17ba781753d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:07,027 INFO [StoreOpener-81761a3d64f4403adedb17ba781753d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81761a3d64f4403adedb17ba781753d2 columnFamilyName A 2024-12-06T16:02:07,027 DEBUG [StoreOpener-81761a3d64f4403adedb17ba781753d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:02:07,027 INFO [StoreOpener-81761a3d64f4403adedb17ba781753d2-1 {}] regionserver.HStore(327): Store=81761a3d64f4403adedb17ba781753d2/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T16:02:07,027 INFO [StoreOpener-81761a3d64f4403adedb17ba781753d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:07,028 INFO [StoreOpener-81761a3d64f4403adedb17ba781753d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81761a3d64f4403adedb17ba781753d2 columnFamilyName B 2024-12-06T16:02:07,028 DEBUG [StoreOpener-81761a3d64f4403adedb17ba781753d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:02:07,029 INFO [StoreOpener-81761a3d64f4403adedb17ba781753d2-1 {}] regionserver.HStore(327): Store=81761a3d64f4403adedb17ba781753d2/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T16:02:07,029 INFO [StoreOpener-81761a3d64f4403adedb17ba781753d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:07,030 INFO [StoreOpener-81761a3d64f4403adedb17ba781753d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81761a3d64f4403adedb17ba781753d2 columnFamilyName C 2024-12-06T16:02:07,030 DEBUG [StoreOpener-81761a3d64f4403adedb17ba781753d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:02:07,030 INFO [StoreOpener-81761a3d64f4403adedb17ba781753d2-1 {}] regionserver.HStore(327): Store=81761a3d64f4403adedb17ba781753d2/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T16:02:07,030 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:07,031 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:07,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-12-06T16:02:07,032 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:07,033 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:07,033 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:07,033 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T16:02:07,034 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:07,036 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T16:02:07,037 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 81761a3d64f4403adedb17ba781753d2; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69065174, jitterRate=0.02915129065513611}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T16:02:07,037 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:07,038 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 81761a3d64f4403adedb17ba781753d2: Running coprocessor pre-open hook at 1733500927024Writing region info on filesystem at 1733500927024Initializing all the Stores at 1733500927025 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733500927025Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733500927025Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733500927025Cleaning up temporary data from old regions at 1733500927033 (+8 ms)Running coprocessor post-open hooks at 1733500927037 (+4 ms)Region opened successfully at 1733500927038 (+1 ms) 2024-12-06T16:02:07,039 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., pid=79, masterSystemTime=1733500927019 2024-12-06T16:02:07,041 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:07,041 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:07,041 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=81761a3d64f4403adedb17ba781753d2, regionState=OPEN, openSeqNum=2, regionLocation=85bef17d9292,46105,1733500827732 2024-12-06T16:02:07,043 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=78, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 because future has completed 2024-12-06T16:02:07,047 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-06T16:02:07,047 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; OpenRegionProcedure 81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 in 177 msec 2024-12-06T16:02:07,049 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=77 2024-12-06T16:02:07,049 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=77, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81761a3d64f4403adedb17ba781753d2, ASSIGN in 335 msec 2024-12-06T16:02:07,049 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T16:02:07,050 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733500927049"}]},"ts":"1733500927049"} 2024-12-06T16:02:07,052 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-06T16:02:07,052 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T16:02:07,054 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1610 sec 2024-12-06T16:02:08,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-12-06T16:02:08,042 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:02:08,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29820261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,043 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:02:08,043 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:02:08,044 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:02:08,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:02:08,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:02:08,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f488d99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:02:08,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:02:08,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:08,046 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:02:08,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27a49b4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,047 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:02:08,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:02:08,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:02:08,049 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:02:08,049 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T16:02:08,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 85bef17d9292,45907,1733500826847 2024-12-06T16:02:08,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@350109aa 2024-12-06T16:02:08,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T16:02:08,051 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46794, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T16:02:08,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-06T16:02:08,052 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T16:02:08,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-06T16:02:08,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742196_1372 (size=995) 2024-12-06T16:02:08,464 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-12-06T16:02:08,464 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-12-06T16:02:08,466 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-06T16:02:08,468 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81761a3d64f4403adedb17ba781753d2, REOPEN/MOVE}] 2024-12-06T16:02:08,469 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81761a3d64f4403adedb17ba781753d2, REOPEN/MOVE 2024-12-06T16:02:08,470 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=81761a3d64f4403adedb17ba781753d2, regionState=CLOSING, regionLocation=85bef17d9292,46105,1733500827732 2024-12-06T16:02:08,471 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=81761a3d64f4403adedb17ba781753d2, REOPEN/MOVE because future has completed 2024-12-06T16:02:08,471 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-06T16:02:08,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732}] 2024-12-06T16:02:08,623 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:08,623 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-06T16:02:08,624 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing 81761a3d64f4403adedb17ba781753d2, disabling compactions & flushes 2024-12-06T16:02:08,624 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:08,624 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:08,624 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. after waiting 0 ms 2024-12-06T16:02:08,624 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:08,627 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-06T16:02:08,628 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:08,628 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for 81761a3d64f4403adedb17ba781753d2: Waiting for close lock at 1733500928624Running coprocessor pre-close hooks at 1733500928624Disabling compacts and flushes for region at 1733500928624Disabling writes for close at 1733500928624Writing region close event to WAL at 1733500928624Running coprocessor post-close hooks at 1733500928628 (+4 ms)Closed at 1733500928628 2024-12-06T16:02:08,628 WARN [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionServer(3299): Not adding moved region record: 81761a3d64f4403adedb17ba781753d2 to self. 2024-12-06T16:02:08,630 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:08,630 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=81761a3d64f4403adedb17ba781753d2, regionState=CLOSED 2024-12-06T16:02:08,631 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 because future has completed 2024-12-06T16:02:08,634 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-06T16:02:08,634 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure 81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 in 161 msec 2024-12-06T16:02:08,635 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=81761a3d64f4403adedb17ba781753d2, REOPEN/MOVE; state=CLOSED, location=85bef17d9292,46105,1733500827732; forceNewPlan=false, retain=true 2024-12-06T16:02:08,785 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=81761a3d64f4403adedb17ba781753d2, regionState=OPENING, regionLocation=85bef17d9292,46105,1733500827732 2024-12-06T16:02:08,787 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=81761a3d64f4403adedb17ba781753d2, REOPEN/MOVE because future has completed 2024-12-06T16:02:08,787 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732}] 2024-12-06T16:02:08,942 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:08,942 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(7752): Opening region: {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} 2024-12-06T16:02:08,943 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:08,943 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T16:02:08,943 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(7794): checking encryption for 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:08,943 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(7797): checking classloading for 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:08,944 INFO [StoreOpener-81761a3d64f4403adedb17ba781753d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:08,945 INFO [StoreOpener-81761a3d64f4403adedb17ba781753d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81761a3d64f4403adedb17ba781753d2 columnFamilyName A 2024-12-06T16:02:08,946 DEBUG [StoreOpener-81761a3d64f4403adedb17ba781753d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:02:08,946 INFO [StoreOpener-81761a3d64f4403adedb17ba781753d2-1 {}] regionserver.HStore(327): Store=81761a3d64f4403adedb17ba781753d2/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T16:02:08,946 INFO [StoreOpener-81761a3d64f4403adedb17ba781753d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:08,947 INFO [StoreOpener-81761a3d64f4403adedb17ba781753d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81761a3d64f4403adedb17ba781753d2 columnFamilyName B 2024-12-06T16:02:08,947 DEBUG [StoreOpener-81761a3d64f4403adedb17ba781753d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:02:08,947 INFO [StoreOpener-81761a3d64f4403adedb17ba781753d2-1 {}] regionserver.HStore(327): Store=81761a3d64f4403adedb17ba781753d2/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T16:02:08,947 INFO [StoreOpener-81761a3d64f4403adedb17ba781753d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:08,948 INFO [StoreOpener-81761a3d64f4403adedb17ba781753d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81761a3d64f4403adedb17ba781753d2 columnFamilyName C 2024-12-06T16:02:08,948 DEBUG [StoreOpener-81761a3d64f4403adedb17ba781753d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:02:08,948 INFO [StoreOpener-81761a3d64f4403adedb17ba781753d2-1 {}] regionserver.HStore(327): Store=81761a3d64f4403adedb17ba781753d2/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T16:02:08,948 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1038): replaying wal for 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:08,949 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:08,949 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:08,950 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1048): stopping wal replay for 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:08,950 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1060): Cleaning up temporary data for 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:08,950 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T16:02:08,951 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1093): writing seq id for 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:08,952 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1114): Opened 81761a3d64f4403adedb17ba781753d2; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64414312, jitterRate=-0.04015195369720459}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T16:02:08,952 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:08,952 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1006): Region open journal for 81761a3d64f4403adedb17ba781753d2: Running coprocessor pre-open hook at 1733500928943Writing region info on filesystem at 1733500928943Initializing all the Stores at 1733500928943Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733500928944 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733500928944Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733500928944Cleaning up temporary data from old regions at 1733500928950 (+6 ms)Running coprocessor post-open hooks at 1733500928952 (+2 ms)Region opened successfully at 1733500928952 2024-12-06T16:02:08,953 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., pid=84, masterSystemTime=1733500928939 2024-12-06T16:02:08,955 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:08,955 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:08,955 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=81761a3d64f4403adedb17ba781753d2, regionState=OPEN, openSeqNum=5, regionLocation=85bef17d9292,46105,1733500827732 2024-12-06T16:02:08,956 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 because future has completed 2024-12-06T16:02:08,958 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=82 2024-12-06T16:02:08,958 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=82, state=SUCCESS, hasLock=false; OpenRegionProcedure 81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 in 170 msec 2024-12-06T16:02:08,960 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-12-06T16:02:08,960 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81761a3d64f4403adedb17ba781753d2, REOPEN/MOVE in 490 msec 2024-12-06T16:02:08,962 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-06T16:02:08,962 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 494 msec 2024-12-06T16:02:08,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 909 msec 2024-12-06T16:02:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-06T16:02:08,964 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:02:08,965 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c07883b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,965 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:02:08,965 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:02:08,966 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:02:08,966 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:02:08,966 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:02:08,967 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25d6d4d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,967 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:02:08,967 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:02:08,967 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:08,968 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:02:08,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@abe49dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,969 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16d62a3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,969 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:02:08,969 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:02:08,970 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:02:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:02:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:02:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4617f33a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:02:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:02:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:08,971 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:02:08,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@221d129b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,972 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6513d18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,972 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:02:08,972 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:02:08,972 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:02:08,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:02:08,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:02:08,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fcbcafc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:02:08,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:02:08,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:08,974 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46838, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:02:08,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c9cc5fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,975 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@510f8e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,975 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:02:08,975 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:02:08,975 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:02:08,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:02:08,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:02:08,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33491413, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:02:08,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:02:08,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:08,977 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46862, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:02:08,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e64152e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74436e44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,977 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:02:08,978 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:02:08,978 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:02:08,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:02:08,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:02:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e1b358e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:02:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:02:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:08,979 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46880, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:02:08,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681d82d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@469d248f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,980 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:02:08,981 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:02:08,981 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:02:08,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:02:08,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:02:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77a0f8d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:02:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:02:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:08,982 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46894, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:02:08,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@344efacd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,983 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e8c61ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,983 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:02:08,983 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:02:08,984 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:02:08,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:02:08,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:02:08,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34396bd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:02:08,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:02:08,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:08,985 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46906, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:02:08,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a12a00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,986 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@147a1007, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,986 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:02:08,986 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:02:08,987 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:02:08,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:02:08,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:02:08,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f319c00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:02:08,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:02:08,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:08,988 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46914, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:02:08,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c96be44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,989 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b4d201d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,989 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:02:08,989 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:02:08,990 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:02:08,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:02:08,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:02:08,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625e3708, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:02:08,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:02:08,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:08,991 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:02:08,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a266ff5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,992 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d2b45c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,992 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:02:08,992 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:02:08,993 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:02:08,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:02:08,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:02:08,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c32cb89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:02:08,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:02:08,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:08,994 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:02:08,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d085e67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:08,995 DEBUG [Thread-1678 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:02:08,995 DEBUG [Thread-1680 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:02:08,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:02:08,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:02:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:02:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:02:08,996 DEBUG [Thread-1693 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:02:08,997 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35890, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:02:08,997 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:02:08,997 DEBUG [Thread-1686 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:02:08,997 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:02:08,997 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35892, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:02:08,998 DEBUG [Thread-1682 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:02:08,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:02:08,998 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35898, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:02:08,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:02:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:02:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:02:08,999 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35912, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:02:09,000 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35924, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:02:09,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5] 2024-12-06T16:02:09,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5] 2024-12-06T16:02:09,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5] 2024-12-06T16:02:09,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5] 2024-12-06T16:02:09,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5] 2024-12-06T16:02:09,002 DEBUG [Thread-1684 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:02:09,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:02:09,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:02:09,004 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35932, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:02:09,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5] 2024-12-06T16:02:09,007 DEBUG [Thread-1695 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:02:09,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:02:09,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:02:09,009 DEBUG [Thread-1689 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:02:09,009 DEBUG [Thread-1697 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:02:09,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:02:09,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:02:09,010 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35946, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:02:09,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:02:09,010 DEBUG [Thread-1691 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:02:09,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:02:09,010 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:02:09,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:02:09,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:02:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:09,012 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:02:09,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81761a3d64f4403adedb17ba781753d2 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-06T16:02:09,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5] 2024-12-06T16:02:09,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5] 2024-12-06T16:02:09,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35958, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:02:09,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35972, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:02:09,014 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:02:09,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5] 2024-12-06T16:02:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees 2024-12-06T16:02:09,016 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:02:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-06T16:02:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5] 2024-12-06T16:02:09,018 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=85, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:02:09,018 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:02:09,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412066e053c0c7b3347ec995d1aa4ce81c3c2_81761a3d64f4403adedb17ba781753d2 is 50, key is test_row_0/A:col10/1733500929012/Put/seqid=0 2024-12-06T16:02:09,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742197_1373 (size=12154) 2024-12-06T16:02:09,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:09,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500989101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:09,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:09,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:09,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500989102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:09,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:09,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:09,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500989102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:09,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:09,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500989102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:09,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500989103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:09,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:09,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:09,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-06T16:02:09,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-06T16:02:09,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:09,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:09,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:09,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:09,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:09,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:09,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500989212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:09,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:09,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500989222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:09,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500989222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:09,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:09,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:09,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500989222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:09,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:09,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500989222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:09,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:09,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-06T16:02:09,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:09,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:09,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:09,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:09,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-06T16:02:09,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500989422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:09,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:09,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:09,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500989431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:09,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:09,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:09,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500989431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:09,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:09,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500989432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:09,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:09,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:09,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500989442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:09,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:09,462 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:02:09,466 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412066e053c0c7b3347ec995d1aa4ce81c3c2_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412066e053c0c7b3347ec995d1aa4ce81c3c2_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:09,466 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/e44d3f2ecfb2477da322fe47fdaa47c1, store: [table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:09,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/e44d3f2ecfb2477da322fe47fdaa47c1 is 175, key is test_row_0/A:col10/1733500929012/Put/seqid=0 2024-12-06T16:02:09,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742198_1374 (size=30951) 2024-12-06T16:02:09,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-06T16:02:09,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:09,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:09,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:09,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:09,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:09,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-06T16:02:09,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:09,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:09,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:09,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:09,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:09,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:09,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-06T16:02:09,693 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T16:02:09,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500989732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:09,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:09,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:09,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500989742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:09,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:09,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:09,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500989752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:09,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:09,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:09,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500989752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:09,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:09,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:09,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500989762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:09,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:09,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:09,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-06T16:02:09,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:09,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:09,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:09,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:09,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:09,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:09,873 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/e44d3f2ecfb2477da322fe47fdaa47c1 2024-12-06T16:02:09,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/fece3394285f43778b8e59362d6471c5 is 50, key is test_row_0/B:col10/1733500929012/Put/seqid=0 2024-12-06T16:02:09,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742199_1375 (size=9657) 2024-12-06T16:02:09,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/fece3394285f43778b8e59362d6471c5 2024-12-06T16:02:09,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-06T16:02:09,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:09,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:09,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:09,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:09,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:09,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:09,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/20f4928bb1384e508acd271c95561857 is 50, key is test_row_0/C:col10/1733500929012/Put/seqid=0 2024-12-06T16:02:09,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742200_1376 (size=9657) 2024-12-06T16:02:09,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/20f4928bb1384e508acd271c95561857 2024-12-06T16:02:09,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/e44d3f2ecfb2477da322fe47fdaa47c1 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/e44d3f2ecfb2477da322fe47fdaa47c1 2024-12-06T16:02:09,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/e44d3f2ecfb2477da322fe47fdaa47c1, entries=150, sequenceid=17, filesize=30.2 K 2024-12-06T16:02:09,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/fece3394285f43778b8e59362d6471c5 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/fece3394285f43778b8e59362d6471c5 2024-12-06T16:02:09,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/fece3394285f43778b8e59362d6471c5, entries=100, sequenceid=17, filesize=9.4 K 2024-12-06T16:02:09,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/20f4928bb1384e508acd271c95561857 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/20f4928bb1384e508acd271c95561857 2024-12-06T16:02:09,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/20f4928bb1384e508acd271c95561857, entries=100, sequenceid=17, filesize=9.4 K 2024-12-06T16:02:09,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 81761a3d64f4403adedb17ba781753d2 in 965ms, sequenceid=17, compaction requested=false 2024-12-06T16:02:09,977 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-06T16:02:09,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:10,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-06T16:02:10,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:10,091 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2902): Flushing 81761a3d64f4403adedb17ba781753d2 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-06T16:02:10,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412062c96ef84c58e4cc39ddd6ddd14f633d4_81761a3d64f4403adedb17ba781753d2 is 50, key is test_row_0/A:col10/1733500929101/Put/seqid=0 2024-12-06T16:02:10,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742201_1377 (size=12154) 2024-12-06T16:02:10,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-06T16:02:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:10,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:10,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500990280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500990281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:10,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:10,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500990283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:10,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500990284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500990286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:10,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:10,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500990401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:10,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:10,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:10,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:10,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500990402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:10,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500990401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:10,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:10,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:10,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500990402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:10,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500990402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:10,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:10,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:10,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:10,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:10,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:02:10,514 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412062c96ef84c58e4cc39ddd6ddd14f633d4_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412062c96ef84c58e4cc39ddd6ddd14f633d4_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:10,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/973c88d96816417f9505b3d7b2d4274c, store: [table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:10,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/973c88d96816417f9505b3d7b2d4274c is 175, key is test_row_0/A:col10/1733500929101/Put/seqid=0 2024-12-06T16:02:10,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742202_1378 (size=30955) 2024-12-06T16:02:10,519 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/973c88d96816417f9505b3d7b2d4274c 2024-12-06T16:02:10,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/a17672fd865c489d9c8bc39f5e85a4c4 is 50, key is test_row_0/B:col10/1733500929101/Put/seqid=0 2024-12-06T16:02:10,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742203_1379 (size=12001) 2024-12-06T16:02:10,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:10,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500990621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:10,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:10,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500990622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:10,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:10,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500990622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:10,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500990622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:10,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500990622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:10,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:10,933 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/a17672fd865c489d9c8bc39f5e85a4c4 2024-12-06T16:02:10,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/00a9f63103c64de397a453390ea8c3a2 is 50, key is test_row_0/C:col10/1733500929101/Put/seqid=0 2024-12-06T16:02:10,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742204_1380 (size=12001) 2024-12-06T16:02:10,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500990942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:10,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:10,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500990942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:10,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:10,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500990942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:10,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:10,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:10,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500990944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:10,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:10,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:10,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500990944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:10,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:10,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:10,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:11,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-06T16:02:11,351 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/00a9f63103c64de397a453390ea8c3a2 2024-12-06T16:02:11,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/973c88d96816417f9505b3d7b2d4274c as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/973c88d96816417f9505b3d7b2d4274c 2024-12-06T16:02:11,359 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/973c88d96816417f9505b3d7b2d4274c, entries=150, sequenceid=40, filesize=30.2 K 2024-12-06T16:02:11,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/a17672fd865c489d9c8bc39f5e85a4c4 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/a17672fd865c489d9c8bc39f5e85a4c4 2024-12-06T16:02:11,364 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/a17672fd865c489d9c8bc39f5e85a4c4, entries=150, sequenceid=40, filesize=11.7 K 2024-12-06T16:02:11,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/00a9f63103c64de397a453390ea8c3a2 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/00a9f63103c64de397a453390ea8c3a2 2024-12-06T16:02:11,368 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/00a9f63103c64de397a453390ea8c3a2, entries=150, sequenceid=40, filesize=11.7 K 2024-12-06T16:02:11,371 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 81761a3d64f4403adedb17ba781753d2 in 1280ms, sequenceid=40, compaction requested=false 2024-12-06T16:02:11,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2603): Flush status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:11,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:11,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=86 2024-12-06T16:02:11,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=86 2024-12-06T16:02:11,374 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-12-06T16:02:11,374 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3540 sec 2024-12-06T16:02:11,375 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees in 2.3600 sec 2024-12-06T16:02:11,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:11,468 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:02:11,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81761a3d64f4403adedb17ba781753d2 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-06T16:02:11,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206641ff027602e4a25b09120af9e3bf9ee_81761a3d64f4403adedb17ba781753d2 is 50, key is test_row_0/A:col10/1733500930284/Put/seqid=0 2024-12-06T16:02:11,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742205_1381 (size=14594) 2024-12-06T16:02:11,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:11,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500991504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:11,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:11,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500991505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:11,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:11,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:11,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:11,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:11,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500991520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:11,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:11,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:11,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:11,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:11,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:11,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:11,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:11,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500991521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:11,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:11,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500991522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:11,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:11,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:11,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:11,538 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:11,538 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:11,538 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:11,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:11,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500991632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:11,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:11,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500991632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:11,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:11,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:11,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:11,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:11,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500991632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:11,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:11,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:11,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:11,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:11,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:11,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:11,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:11,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500991643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:11,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:11,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500991643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:11,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:11,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:11,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:11,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:11,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:11,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:11,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500991852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:11,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:11,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:11,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500991852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:11,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:11,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:11,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500991852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:11,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:11,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:11,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:11,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:11,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:11,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:11,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:11,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500991861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:11,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:11,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:11,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:11,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500991862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:11,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:11,889 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:02:11,893 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206641ff027602e4a25b09120af9e3bf9ee_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206641ff027602e4a25b09120af9e3bf9ee_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:11,894 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/1fbdd0db3c8544be8527c5477e70bf36, store: [table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:11,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/1fbdd0db3c8544be8527c5477e70bf36 is 175, key is test_row_0/A:col10/1733500930284/Put/seqid=0 2024-12-06T16:02:11,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742206_1382 (size=39549) 2024-12-06T16:02:12,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500992172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:12,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:12,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:12,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500992172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:12,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:12,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500992172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:12,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:12,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:12,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:12,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:12,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:12,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:12,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500992181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:12,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500992182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:12,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:12,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:12,300 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/1fbdd0db3c8544be8527c5477e70bf36 2024-12-06T16:02:12,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/b35322c397ba4b04b5e7ac3f402bf6bb is 50, key is test_row_0/B:col10/1733500930284/Put/seqid=0 2024-12-06T16:02:12,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742207_1383 (size=12001) 2024-12-06T16:02:12,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:12,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500992691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:12,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:12,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500992691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:12,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:12,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:12,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:12,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:12,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:12,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:12,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:12,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500992692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:12,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:12,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:12,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:12,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:12,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500992692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:12,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:12,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:12,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:12,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:12,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500992702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:12,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:12,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:12,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:12,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/b35322c397ba4b04b5e7ac3f402bf6bb 2024-12-06T16:02:12,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/bc89ac95337f401ab1da32d00f95d78a is 50, key is test_row_0/C:col10/1733500930284/Put/seqid=0 2024-12-06T16:02:12,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742208_1384 (size=12001) 2024-12-06T16:02:12,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/bc89ac95337f401ab1da32d00f95d78a 2024-12-06T16:02:12,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/1fbdd0db3c8544be8527c5477e70bf36 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/1fbdd0db3c8544be8527c5477e70bf36 2024-12-06T16:02:12,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/1fbdd0db3c8544be8527c5477e70bf36, entries=200, sequenceid=54, filesize=38.6 K 2024-12-06T16:02:12,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/b35322c397ba4b04b5e7ac3f402bf6bb as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/b35322c397ba4b04b5e7ac3f402bf6bb 2024-12-06T16:02:12,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/b35322c397ba4b04b5e7ac3f402bf6bb, entries=150, sequenceid=54, filesize=11.7 K 2024-12-06T16:02:12,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/bc89ac95337f401ab1da32d00f95d78a as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/bc89ac95337f401ab1da32d00f95d78a 2024-12-06T16:02:12,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/bc89ac95337f401ab1da32d00f95d78a, entries=150, sequenceid=54, filesize=11.7 K 2024-12-06T16:02:12,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 81761a3d64f4403adedb17ba781753d2 in 1286ms, sequenceid=54, compaction requested=true 2024-12-06T16:02:12,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:12,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81761a3d64f4403adedb17ba781753d2:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:02:12,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:12,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81761a3d64f4403adedb17ba781753d2:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:02:12,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:12,754 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:12,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81761a3d64f4403adedb17ba781753d2:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:02:12,755 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:12,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:12,755 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:12,755 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101455 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:12,755 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 81761a3d64f4403adedb17ba781753d2/B is initiating minor compaction (all files) 2024-12-06T16:02:12,755 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 81761a3d64f4403adedb17ba781753d2/A is initiating minor compaction (all files) 2024-12-06T16:02:12,756 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81761a3d64f4403adedb17ba781753d2/B in TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:12,756 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81761a3d64f4403adedb17ba781753d2/A in TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:12,756 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/fece3394285f43778b8e59362d6471c5, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/a17672fd865c489d9c8bc39f5e85a4c4, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/b35322c397ba4b04b5e7ac3f402bf6bb] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp, totalSize=32.9 K 2024-12-06T16:02:12,756 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/e44d3f2ecfb2477da322fe47fdaa47c1, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/973c88d96816417f9505b3d7b2d4274c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/1fbdd0db3c8544be8527c5477e70bf36] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp, totalSize=99.1 K 2024-12-06T16:02:12,756 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:12,756 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. files: [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/e44d3f2ecfb2477da322fe47fdaa47c1, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/973c88d96816417f9505b3d7b2d4274c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/1fbdd0db3c8544be8527c5477e70bf36] 2024-12-06T16:02:12,756 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting fece3394285f43778b8e59362d6471c5, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733500929011 2024-12-06T16:02:12,756 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting e44d3f2ecfb2477da322fe47fdaa47c1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733500929010 2024-12-06T16:02:12,757 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting a17672fd865c489d9c8bc39f5e85a4c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733500929099 2024-12-06T16:02:12,757 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 973c88d96816417f9505b3d7b2d4274c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733500929099 2024-12-06T16:02:12,757 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting b35322c397ba4b04b5e7ac3f402bf6bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733500930283 2024-12-06T16:02:12,757 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fbdd0db3c8544be8527c5477e70bf36, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733500930271 2024-12-06T16:02:12,764 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81761a3d64f4403adedb17ba781753d2#B#compaction#330 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:12,764 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:12,764 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/722de527f72044b5b31d5b778289a401 is 50, key is test_row_0/B:col10/1733500930284/Put/seqid=0 2024-12-06T16:02:12,766 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206f8a3c79acff84027a8d30509001b0fe9_81761a3d64f4403adedb17ba781753d2 store=[table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:12,768 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206f8a3c79acff84027a8d30509001b0fe9_81761a3d64f4403adedb17ba781753d2, store=[table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:12,768 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206f8a3c79acff84027a8d30509001b0fe9_81761a3d64f4403adedb17ba781753d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:12,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742209_1385 (size=12104) 2024-12-06T16:02:12,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742210_1386 (size=4469) 2024-12-06T16:02:12,795 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81761a3d64f4403adedb17ba781753d2#A#compaction#331 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:12,796 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/e96ee7e94acf4dd99ca423a16be2b9a2 is 175, key is test_row_0/A:col10/1733500930284/Put/seqid=0 2024-12-06T16:02:12,797 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/722de527f72044b5b31d5b778289a401 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/722de527f72044b5b31d5b778289a401 2024-12-06T16:02:12,805 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81761a3d64f4403adedb17ba781753d2/B of 81761a3d64f4403adedb17ba781753d2 into 722de527f72044b5b31d5b778289a401(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:12,805 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:12,805 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., storeName=81761a3d64f4403adedb17ba781753d2/B, priority=13, startTime=1733500932754; duration=0sec 2024-12-06T16:02:12,805 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:12,805 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81761a3d64f4403adedb17ba781753d2:B 2024-12-06T16:02:12,805 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:12,806 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:12,806 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 81761a3d64f4403adedb17ba781753d2/C is initiating minor compaction (all files) 2024-12-06T16:02:12,806 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81761a3d64f4403adedb17ba781753d2/C in TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:12,806 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/20f4928bb1384e508acd271c95561857, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/00a9f63103c64de397a453390ea8c3a2, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/bc89ac95337f401ab1da32d00f95d78a] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp, totalSize=32.9 K 2024-12-06T16:02:12,807 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 20f4928bb1384e508acd271c95561857, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733500929011 2024-12-06T16:02:12,807 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 00a9f63103c64de397a453390ea8c3a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733500929099 2024-12-06T16:02:12,807 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting bc89ac95337f401ab1da32d00f95d78a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733500930283 2024-12-06T16:02:12,817 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81761a3d64f4403adedb17ba781753d2#C#compaction#332 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:12,817 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/82ac452b72604396afaf2e0423cf81b7 is 50, key is test_row_0/C:col10/1733500930284/Put/seqid=0 2024-12-06T16:02:12,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742211_1387 (size=31058) 2024-12-06T16:02:12,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742212_1388 (size=12104) 2024-12-06T16:02:12,842 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/e96ee7e94acf4dd99ca423a16be2b9a2 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/e96ee7e94acf4dd99ca423a16be2b9a2 2024-12-06T16:02:12,852 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/82ac452b72604396afaf2e0423cf81b7 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/82ac452b72604396afaf2e0423cf81b7 2024-12-06T16:02:12,858 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81761a3d64f4403adedb17ba781753d2/C of 81761a3d64f4403adedb17ba781753d2 into 82ac452b72604396afaf2e0423cf81b7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:12,858 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:12,858 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., storeName=81761a3d64f4403adedb17ba781753d2/C, priority=13, startTime=1733500932754; duration=0sec 2024-12-06T16:02:12,858 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:12,858 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81761a3d64f4403adedb17ba781753d2:C 2024-12-06T16:02:12,867 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81761a3d64f4403adedb17ba781753d2/A of 81761a3d64f4403adedb17ba781753d2 into e96ee7e94acf4dd99ca423a16be2b9a2(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:12,867 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:12,867 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., storeName=81761a3d64f4403adedb17ba781753d2/A, priority=13, startTime=1733500932754; duration=0sec 2024-12-06T16:02:12,867 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:12,867 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81761a3d64f4403adedb17ba781753d2:A 2024-12-06T16:02:13,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-06T16:02:13,173 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:02:13,173 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:02:13,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees 2024-12-06T16:02:13,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-06T16:02:13,175 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:02:13,176 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:02:13,176 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:02:13,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-06T16:02:13,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-12-06T16:02:13,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:13,329 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2902): Flushing 81761a3d64f4403adedb17ba781753d2 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-06T16:02:13,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412062d932b68e39b42fdbd3ff6074a149a0e_81761a3d64f4403adedb17ba781753d2 is 50, key is test_row_0/A:col10/1733500931521/Put/seqid=0 2024-12-06T16:02:13,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742213_1389 (size=12154) 2024-12-06T16:02:13,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-06T16:02:13,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:13,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:13,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:02:13,745 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412062d932b68e39b42fdbd3ff6074a149a0e_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412062d932b68e39b42fdbd3ff6074a149a0e_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:13,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/17f7452fa5ba48579b8c097a7ae0a162, store: [table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:13,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/17f7452fa5ba48579b8c097a7ae0a162 is 175, key is test_row_0/A:col10/1733500931521/Put/seqid=0 2024-12-06T16:02:13,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:13,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500993736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:13,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:13,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500993736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:13,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:13,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:13,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:13,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:13,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500993743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:13,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:13,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:13,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:13,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:13,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:13,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:13,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742214_1390 (size=30955) 2024-12-06T16:02:13,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:13,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500993749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:13,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:13,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:13,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:13,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:13,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500993749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:13,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-06T16:02:13,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:13,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500993852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:13,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:13,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500993852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:13,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:13,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500993853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:13,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:13,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500993862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:13,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:13,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:13,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:13,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:13,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500993872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:13,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:13,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:13,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:14,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:14,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500994072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:14,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500994072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:14,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500994072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:14,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:14,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:14,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:14,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500994082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:14,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:14,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:14,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:14,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:14,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500994083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:14,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:14,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:14,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:14,153 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/17f7452fa5ba48579b8c097a7ae0a162 2024-12-06T16:02:14,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/6db4ae9873e847699542de1de033c87d is 50, key is test_row_0/B:col10/1733500931521/Put/seqid=0 2024-12-06T16:02:14,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742215_1391 (size=12001) 2024-12-06T16:02:14,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-06T16:02:14,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:14,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500994392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:14,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:14,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500994392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:14,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:14,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500994392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:14,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:14,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500994401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:14,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:14,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:14,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500994401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:14,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:14,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:14,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:14,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:14,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:14,564 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/6db4ae9873e847699542de1de033c87d 2024-12-06T16:02:14,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/5d1f9c3b06b0481bab645af64e3b54be is 50, key is test_row_0/C:col10/1733500931521/Put/seqid=0 2024-12-06T16:02:14,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742216_1392 (size=12001) 2024-12-06T16:02:14,579 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/5d1f9c3b06b0481bab645af64e3b54be 2024-12-06T16:02:14,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/17f7452fa5ba48579b8c097a7ae0a162 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/17f7452fa5ba48579b8c097a7ae0a162 2024-12-06T16:02:14,588 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/17f7452fa5ba48579b8c097a7ae0a162, entries=150, sequenceid=79, filesize=30.2 K 2024-12-06T16:02:14,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/6db4ae9873e847699542de1de033c87d as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/6db4ae9873e847699542de1de033c87d 2024-12-06T16:02:14,593 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/6db4ae9873e847699542de1de033c87d, entries=150, sequenceid=79, filesize=11.7 K 2024-12-06T16:02:14,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/5d1f9c3b06b0481bab645af64e3b54be as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/5d1f9c3b06b0481bab645af64e3b54be 2024-12-06T16:02:14,598 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/5d1f9c3b06b0481bab645af64e3b54be, entries=150, sequenceid=79, filesize=11.7 K 2024-12-06T16:02:14,599 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 81761a3d64f4403adedb17ba781753d2 in 1270ms, sequenceid=79, compaction requested=false 2024-12-06T16:02:14,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2603): Flush status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:14,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:14,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-12-06T16:02:14,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=88 2024-12-06T16:02:14,602 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-12-06T16:02:14,602 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4240 sec 2024-12-06T16:02:14,604 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees in 1.4300 sec 2024-12-06T16:02:14,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:14,909 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:02:14,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81761a3d64f4403adedb17ba781753d2 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-06T16:02:14,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206f66c83e1557146d180ff7bca7feecebe_81761a3d64f4403adedb17ba781753d2 is 50, key is test_row_0/A:col10/1733500933740/Put/seqid=0 2024-12-06T16:02:14,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742217_1393 (size=14594) 2024-12-06T16:02:14,936 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:02:14,941 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206f66c83e1557146d180ff7bca7feecebe_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206f66c83e1557146d180ff7bca7feecebe_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:14,942 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/b81ecf0d5db64dce962b502185fc5d45, store: [table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:14,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/b81ecf0d5db64dce962b502185fc5d45 is 175, key is test_row_0/A:col10/1733500933740/Put/seqid=0 2024-12-06T16:02:14,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742218_1394 (size=39549) 2024-12-06T16:02:14,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:14,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500994955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:14,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:14,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500994968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:14,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:14,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500994969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:14,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:14,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500994969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:14,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:14,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:14,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:14,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:14,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500994970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:14,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:14,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:14,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:15,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:15,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500995072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:15,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:15,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:15,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:15,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:15,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500995072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:15,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:15,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:15,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:15,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:15,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500995082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:15,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:15,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:15,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:15,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500995091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:15,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:15,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:15,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:15,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500995091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:15,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:15,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:15,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:15,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:15,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500995292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:15,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:15,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500995292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:15,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:15,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:15,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:15,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:15,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:15,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:15,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500995292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:15,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:15,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:15,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:15,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:15,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:15,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500995312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:15,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:15,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500995312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:15,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-06T16:02:15,322 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:02:15,323 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:02:15,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-12-06T16:02:15,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-06T16:02:15,325 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:02:15,325 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:02:15,325 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:02:15,349 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/b81ecf0d5db64dce962b502185fc5d45 2024-12-06T16:02:15,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/65e57d05727642c094ffbc91085296e0 is 50, key is test_row_0/B:col10/1733500933740/Put/seqid=0 2024-12-06T16:02:15,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742219_1395 (size=12001) 2024-12-06T16:02:15,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-06T16:02:15,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-06T16:02:15,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:15,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:15,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:15,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:15,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:15,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:15,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:15,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500995611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:15,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:15,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:15,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:15,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:15,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500995612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:15,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:15,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:15,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:15,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:15,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500995612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:15,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:15,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:15,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:15,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-06T16:02:15,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:15,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:15,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:15,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:15,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-06T16:02:15,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:15,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500995632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:15,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:15,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500995632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:15,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:15,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:15,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:15,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:15,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:15,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:15,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/65e57d05727642c094ffbc91085296e0 2024-12-06T16:02:15,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/6f2f509c81014f02ad70d84f143c7d82 is 50, key is test_row_0/C:col10/1733500933740/Put/seqid=0 2024-12-06T16:02:15,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-06T16:02:15,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:15,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:15,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:15,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742220_1396 (size=12001) 2024-12-06T16:02:15,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:15,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:15,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/6f2f509c81014f02ad70d84f143c7d82 2024-12-06T16:02:15,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:15,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/b81ecf0d5db64dce962b502185fc5d45 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/b81ecf0d5db64dce962b502185fc5d45 2024-12-06T16:02:15,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/b81ecf0d5db64dce962b502185fc5d45, entries=200, sequenceid=95, filesize=38.6 K 2024-12-06T16:02:15,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/65e57d05727642c094ffbc91085296e0 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/65e57d05727642c094ffbc91085296e0 2024-12-06T16:02:15,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/65e57d05727642c094ffbc91085296e0, entries=150, sequenceid=95, filesize=11.7 K 2024-12-06T16:02:15,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/6f2f509c81014f02ad70d84f143c7d82 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/6f2f509c81014f02ad70d84f143c7d82 2024-12-06T16:02:15,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/6f2f509c81014f02ad70d84f143c7d82, entries=150, sequenceid=95, filesize=11.7 K 2024-12-06T16:02:15,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 81761a3d64f4403adedb17ba781753d2 in 897ms, sequenceid=95, compaction requested=true 2024-12-06T16:02:15,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:15,807 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:15,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81761a3d64f4403adedb17ba781753d2:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:02:15,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:15,807 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:15,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81761a3d64f4403adedb17ba781753d2:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:02:15,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:15,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81761a3d64f4403adedb17ba781753d2:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:02:15,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:15,808 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:15,808 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:15,808 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 81761a3d64f4403adedb17ba781753d2/A is initiating minor compaction (all files) 2024-12-06T16:02:15,808 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 81761a3d64f4403adedb17ba781753d2/B is initiating minor compaction (all files) 2024-12-06T16:02:15,809 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81761a3d64f4403adedb17ba781753d2/B in TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:15,809 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81761a3d64f4403adedb17ba781753d2/A in TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:15,809 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/e96ee7e94acf4dd99ca423a16be2b9a2, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/17f7452fa5ba48579b8c097a7ae0a162, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/b81ecf0d5db64dce962b502185fc5d45] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp, totalSize=99.2 K 2024-12-06T16:02:15,809 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/722de527f72044b5b31d5b778289a401, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/6db4ae9873e847699542de1de033c87d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/65e57d05727642c094ffbc91085296e0] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp, totalSize=35.3 K 2024-12-06T16:02:15,809 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:15,809 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. files: [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/e96ee7e94acf4dd99ca423a16be2b9a2, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/17f7452fa5ba48579b8c097a7ae0a162, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/b81ecf0d5db64dce962b502185fc5d45] 2024-12-06T16:02:15,809 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 722de527f72044b5b31d5b778289a401, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733500930283 2024-12-06T16:02:15,809 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting e96ee7e94acf4dd99ca423a16be2b9a2, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733500930283 2024-12-06T16:02:15,809 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 6db4ae9873e847699542de1de033c87d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733500931473 2024-12-06T16:02:15,809 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17f7452fa5ba48579b8c097a7ae0a162, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733500931473 2024-12-06T16:02:15,810 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 65e57d05727642c094ffbc91085296e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733500933735 2024-12-06T16:02:15,810 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting b81ecf0d5db64dce962b502185fc5d45, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733500933735 2024-12-06T16:02:15,822 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:15,824 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81761a3d64f4403adedb17ba781753d2#B#compaction#340 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:15,825 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/b80e3a0c72f84a7ba188be567764005a is 50, key is test_row_0/B:col10/1733500933740/Put/seqid=0 2024-12-06T16:02:15,827 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120629b0b22f89e943f3b2503aab274ddbaf_81761a3d64f4403adedb17ba781753d2 store=[table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:15,828 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120629b0b22f89e943f3b2503aab274ddbaf_81761a3d64f4403adedb17ba781753d2, store=[table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:15,828 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120629b0b22f89e943f3b2503aab274ddbaf_81761a3d64f4403adedb17ba781753d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:15,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742221_1397 (size=12207) 2024-12-06T16:02:15,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742222_1398 (size=4469) 2024-12-06T16:02:15,849 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81761a3d64f4403adedb17ba781753d2#A#compaction#339 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:15,849 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/3d517f61816f4fbb9c74ffef162c4e6d is 175, key is test_row_0/A:col10/1733500933740/Put/seqid=0 2024-12-06T16:02:15,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742223_1399 (size=31161) 2024-12-06T16:02:15,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-06T16:02:15,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:15,937 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing 81761a3d64f4403adedb17ba781753d2 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-06T16:02:15,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206a1b1b213145a4cfd8000a5aa338cc3ca_81761a3d64f4403adedb17ba781753d2 is 50, key is test_row_0/A:col10/1733500934966/Put/seqid=0 2024-12-06T16:02:15,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742224_1400 (size=12154) 2024-12-06T16:02:15,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-06T16:02:16,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:16,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:16,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:16,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500996154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:16,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:16,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:16,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500996156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:16,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:16,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:16,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500996164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:16,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:16,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500996165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:16,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:16,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500996166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:16,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:16,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:16,243 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/b80e3a0c72f84a7ba188be567764005a as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/b80e3a0c72f84a7ba188be567764005a 2024-12-06T16:02:16,253 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81761a3d64f4403adedb17ba781753d2/B of 81761a3d64f4403adedb17ba781753d2 into b80e3a0c72f84a7ba188be567764005a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:16,253 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:16,253 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., storeName=81761a3d64f4403adedb17ba781753d2/B, priority=13, startTime=1733500935807; duration=0sec 2024-12-06T16:02:16,254 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:16,254 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81761a3d64f4403adedb17ba781753d2:B 2024-12-06T16:02:16,254 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:16,255 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:16,255 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 81761a3d64f4403adedb17ba781753d2/C is initiating minor compaction (all files) 2024-12-06T16:02:16,255 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81761a3d64f4403adedb17ba781753d2/C in TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:16,255 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/82ac452b72604396afaf2e0423cf81b7, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/5d1f9c3b06b0481bab645af64e3b54be, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/6f2f509c81014f02ad70d84f143c7d82] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp, totalSize=35.3 K 2024-12-06T16:02:16,256 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 82ac452b72604396afaf2e0423cf81b7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733500930283 2024-12-06T16:02:16,257 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d1f9c3b06b0481bab645af64e3b54be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733500931473 2024-12-06T16:02:16,257 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f2f509c81014f02ad70d84f143c7d82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733500933735 2024-12-06T16:02:16,258 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/3d517f61816f4fbb9c74ffef162c4e6d as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/3d517f61816f4fbb9c74ffef162c4e6d 2024-12-06T16:02:16,264 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81761a3d64f4403adedb17ba781753d2/A of 81761a3d64f4403adedb17ba781753d2 into 3d517f61816f4fbb9c74ffef162c4e6d(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:16,264 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:16,264 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., storeName=81761a3d64f4403adedb17ba781753d2/A, priority=13, startTime=1733500935807; duration=0sec 2024-12-06T16:02:16,264 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:16,264 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81761a3d64f4403adedb17ba781753d2:A 2024-12-06T16:02:16,265 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81761a3d64f4403adedb17ba781753d2#C#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:16,265 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/ea1f00667e1f4997ba2f8b59579bd4e2 is 50, key is test_row_0/C:col10/1733500933740/Put/seqid=0 2024-12-06T16:02:16,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742225_1401 (size=12207) 2024-12-06T16:02:16,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500996272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:16,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:16,275 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/ea1f00667e1f4997ba2f8b59579bd4e2 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/ea1f00667e1f4997ba2f8b59579bd4e2 2024-12-06T16:02:16,280 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81761a3d64f4403adedb17ba781753d2/C of 81761a3d64f4403adedb17ba781753d2 into ea1f00667e1f4997ba2f8b59579bd4e2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:16,280 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:16,280 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., storeName=81761a3d64f4403adedb17ba781753d2/C, priority=13, startTime=1733500935807; duration=0sec 2024-12-06T16:02:16,280 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:16,280 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81761a3d64f4403adedb17ba781753d2:C 2024-12-06T16:02:16,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500996282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:16,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500996282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:16,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500996282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:16,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500996282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:16,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:02:16,354 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206a1b1b213145a4cfd8000a5aa338cc3ca_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a1b1b213145a4cfd8000a5aa338cc3ca_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:16,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/ed772275f8464d0e82ede86b31bdf162, store: [table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:16,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/ed772275f8464d0e82ede86b31bdf162 is 175, key is test_row_0/A:col10/1733500934966/Put/seqid=0 2024-12-06T16:02:16,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742226_1402 (size=30955) 2024-12-06T16:02:16,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-06T16:02:16,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500996482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:16,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:16,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500996501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:16,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:16,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500996501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:16,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:16,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:16,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500996501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:16,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500996502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:16,760 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/ed772275f8464d0e82ede86b31bdf162 2024-12-06T16:02:16,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/614d29bd6ac546598a7fdc5d8405cd5e is 50, key is test_row_0/B:col10/1733500934966/Put/seqid=0 2024-12-06T16:02:16,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742227_1403 (size=12001) 2024-12-06T16:02:16,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:16,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500996791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:16,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:16,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:16,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500996812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:16,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:16,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500996812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:16,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:16,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:16,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500996812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:16,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:16,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:16,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:16,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500996821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:16,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:16,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:17,191 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/614d29bd6ac546598a7fdc5d8405cd5e 2024-12-06T16:02:17,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/b6ca1d1adc9948efbeb24c4ba6127959 is 50, key is test_row_0/C:col10/1733500934966/Put/seqid=0 2024-12-06T16:02:17,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742228_1404 (size=12001) 2024-12-06T16:02:17,231 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/b6ca1d1adc9948efbeb24c4ba6127959 2024-12-06T16:02:17,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/ed772275f8464d0e82ede86b31bdf162 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/ed772275f8464d0e82ede86b31bdf162 2024-12-06T16:02:17,244 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/ed772275f8464d0e82ede86b31bdf162, entries=150, sequenceid=116, filesize=30.2 K 2024-12-06T16:02:17,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/614d29bd6ac546598a7fdc5d8405cd5e as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/614d29bd6ac546598a7fdc5d8405cd5e 2024-12-06T16:02:17,259 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/614d29bd6ac546598a7fdc5d8405cd5e, entries=150, sequenceid=116, filesize=11.7 K 2024-12-06T16:02:17,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/b6ca1d1adc9948efbeb24c4ba6127959 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/b6ca1d1adc9948efbeb24c4ba6127959 2024-12-06T16:02:17,264 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/b6ca1d1adc9948efbeb24c4ba6127959, entries=150, sequenceid=116, filesize=11.7 K 2024-12-06T16:02:17,265 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 81761a3d64f4403adedb17ba781753d2 in 1328ms, sequenceid=116, compaction requested=false 2024-12-06T16:02:17,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:17,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:17,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-12-06T16:02:17,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-12-06T16:02:17,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-06T16:02:17,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9420 sec 2024-12-06T16:02:17,279 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 1.9540 sec 2024-12-06T16:02:17,324 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:02:17,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81761a3d64f4403adedb17ba781753d2 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-06T16:02:17,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:17,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206822156c6e814433a9cad5aca4faba897_81761a3d64f4403adedb17ba781753d2 is 50, key is test_row_0/A:col10/1733500936151/Put/seqid=0 2024-12-06T16:02:17,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742229_1405 (size=14744) 2024-12-06T16:02:17,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:17,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:17,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500997379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:17,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500997373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:17,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:17,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:17,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:17,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:17,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:17,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:17,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:17,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500997393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:17,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:17,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:17,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:17,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:17,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500997396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:17,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:17,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:17,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:17,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:17,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500997399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:17,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:17,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:17,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:17,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-06T16:02:17,473 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:02:17,473 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:02:17,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-12-06T16:02:17,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-06T16:02:17,475 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:02:17,476 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:02:17,476 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:02:17,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:17,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500997502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:17,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:17,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:17,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:17,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:17,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500997512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:17,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:17,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:17,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:17,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:17,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500997522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:17,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:17,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:17,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:17,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:17,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500997522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:17,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:17,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:17,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:17,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:17,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500997522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:17,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:17,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:17,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:17,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-06T16:02:17,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-06T16:02:17,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:17,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:17,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:17,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:17,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:17,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:17,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:17,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500997723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:17,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:17,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:17,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:17,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:17,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500997732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:17,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:17,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:17,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:17,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:17,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500997741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:17,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:17,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500997742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:17,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:17,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:17,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:17,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:17,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:17,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:17,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500997742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:17,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:17,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:17,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:17,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:17,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-06T16:02:17,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:17,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:17,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:17,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:17,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:17,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:17,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-06T16:02:17,799 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:02:17,804 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206822156c6e814433a9cad5aca4faba897_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206822156c6e814433a9cad5aca4faba897_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:17,806 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/c95f3c41067840368d9280f00e4783a3, store: [table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:17,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/c95f3c41067840368d9280f00e4783a3 is 175, key is test_row_0/A:col10/1733500936151/Put/seqid=0 2024-12-06T16:02:17,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742230_1406 (size=39699) 2024-12-06T16:02:17,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-06T16:02:17,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:17,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:17,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:17,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:17,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:18,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:18,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500998042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:18,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:18,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:18,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:18,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:18,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500998051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:18,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:18,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:18,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:18,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:18,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500998062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:18,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:18,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:18,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:18,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:18,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500998062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:18,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:18,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:18,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:18,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:18,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500998062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:18,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:18,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:18,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:18,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-06T16:02:18,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:18,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:18,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:18,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:18,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:18,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:18,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-06T16:02:18,212 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/c95f3c41067840368d9280f00e4783a3 2024-12-06T16:02:18,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/1f4f4245343d40baa654015fad5d4915 is 50, key is test_row_0/B:col10/1733500936151/Put/seqid=0 2024-12-06T16:02:18,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-06T16:02:18,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:18,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:18,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:18,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:18,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:18,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742231_1407 (size=12101) 2024-12-06T16:02:18,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-06T16:02:18,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:18,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:18,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:18,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:18,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:18,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:18,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-06T16:02:18,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:18,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:18,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:18,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:18,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:18,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:18,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500998561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:18,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:18,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:18,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:18,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:18,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500998572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:18,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:18,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:18,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:18,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500998582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:18,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:18,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500998582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:18,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:18,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500998582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:18,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:18,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:18,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-06T16:02:18,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/1f4f4245343d40baa654015fad5d4915 2024-12-06T16:02:18,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/fa940d57dea64247b64848ae358e02ac is 50, key is test_row_0/C:col10/1733500936151/Put/seqid=0 2024-12-06T16:02:18,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742232_1408 (size=12101) 2024-12-06T16:02:18,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-06T16:02:18,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:18,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:18,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:18,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:18,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:18,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-06T16:02:18,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:18,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:18,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:18,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:18,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:19,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-06T16:02:19,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:19,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:19,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:19,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:19,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:19,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:19,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/fa940d57dea64247b64848ae358e02ac 2024-12-06T16:02:19,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/c95f3c41067840368d9280f00e4783a3 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/c95f3c41067840368d9280f00e4783a3 2024-12-06T16:02:19,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/c95f3c41067840368d9280f00e4783a3, entries=200, sequenceid=135, filesize=38.8 K 2024-12-06T16:02:19,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/1f4f4245343d40baa654015fad5d4915 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/1f4f4245343d40baa654015fad5d4915 2024-12-06T16:02:19,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/1f4f4245343d40baa654015fad5d4915, entries=150, sequenceid=135, filesize=11.8 K 2024-12-06T16:02:19,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/fa940d57dea64247b64848ae358e02ac as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/fa940d57dea64247b64848ae358e02ac 2024-12-06T16:02:19,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/fa940d57dea64247b64848ae358e02ac, entries=150, sequenceid=135, filesize=11.8 K 2024-12-06T16:02:19,090 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 81761a3d64f4403adedb17ba781753d2 in 1765ms, sequenceid=135, compaction requested=true 2024-12-06T16:02:19,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:19,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81761a3d64f4403adedb17ba781753d2:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:02:19,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:19,090 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:19,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81761a3d64f4403adedb17ba781753d2:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:02:19,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:19,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81761a3d64f4403adedb17ba781753d2:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:02:19,090 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:19,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:19,091 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:19,091 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 81761a3d64f4403adedb17ba781753d2/B is initiating minor compaction (all files) 2024-12-06T16:02:19,091 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81761a3d64f4403adedb17ba781753d2/B in TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:19,091 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/b80e3a0c72f84a7ba188be567764005a, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/614d29bd6ac546598a7fdc5d8405cd5e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/1f4f4245343d40baa654015fad5d4915] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp, totalSize=35.5 K 2024-12-06T16:02:19,091 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101815 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:19,091 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 81761a3d64f4403adedb17ba781753d2/A is initiating minor compaction (all files) 2024-12-06T16:02:19,091 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81761a3d64f4403adedb17ba781753d2/A in TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:19,091 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/3d517f61816f4fbb9c74ffef162c4e6d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/ed772275f8464d0e82ede86b31bdf162, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/c95f3c41067840368d9280f00e4783a3] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp, totalSize=99.4 K 2024-12-06T16:02:19,091 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:19,092 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. files: [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/3d517f61816f4fbb9c74ffef162c4e6d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/ed772275f8464d0e82ede86b31bdf162, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/c95f3c41067840368d9280f00e4783a3] 2024-12-06T16:02:19,092 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting b80e3a0c72f84a7ba188be567764005a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733500933735 2024-12-06T16:02:19,092 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d517f61816f4fbb9c74ffef162c4e6d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733500933735 2024-12-06T16:02:19,092 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 614d29bd6ac546598a7fdc5d8405cd5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733500934966 2024-12-06T16:02:19,093 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f4f4245343d40baa654015fad5d4915, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733500936151 2024-12-06T16:02:19,093 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed772275f8464d0e82ede86b31bdf162, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733500934966 2024-12-06T16:02:19,094 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting c95f3c41067840368d9280f00e4783a3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733500936151 2024-12-06T16:02:19,103 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:19,105 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81761a3d64f4403adedb17ba781753d2#B#compaction#349 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:19,106 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/9df178cc4f3f4963871b88fc2e71591e is 50, key is test_row_0/B:col10/1733500936151/Put/seqid=0 2024-12-06T16:02:19,108 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206500971094c3b41f0a69edb4052d3b8a2_81761a3d64f4403adedb17ba781753d2 store=[table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:19,109 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206500971094c3b41f0a69edb4052d3b8a2_81761a3d64f4403adedb17ba781753d2, store=[table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:19,109 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206500971094c3b41f0a69edb4052d3b8a2_81761a3d64f4403adedb17ba781753d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:19,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742233_1409 (size=12409) 2024-12-06T16:02:19,117 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/9df178cc4f3f4963871b88fc2e71591e as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/9df178cc4f3f4963871b88fc2e71591e 2024-12-06T16:02:19,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742234_1410 (size=4469) 2024-12-06T16:02:19,125 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81761a3d64f4403adedb17ba781753d2/B of 81761a3d64f4403adedb17ba781753d2 into 9df178cc4f3f4963871b88fc2e71591e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:19,125 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:19,125 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., storeName=81761a3d64f4403adedb17ba781753d2/B, priority=13, startTime=1733500939090; duration=0sec 2024-12-06T16:02:19,125 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:19,125 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81761a3d64f4403adedb17ba781753d2:B 2024-12-06T16:02:19,125 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:19,129 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:19,129 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 81761a3d64f4403adedb17ba781753d2/C is initiating minor compaction (all files) 2024-12-06T16:02:19,129 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81761a3d64f4403adedb17ba781753d2/C in TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:19,129 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/ea1f00667e1f4997ba2f8b59579bd4e2, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/b6ca1d1adc9948efbeb24c4ba6127959, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/fa940d57dea64247b64848ae358e02ac] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp, totalSize=35.5 K 2024-12-06T16:02:19,129 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting ea1f00667e1f4997ba2f8b59579bd4e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733500933735 2024-12-06T16:02:19,129 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting b6ca1d1adc9948efbeb24c4ba6127959, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733500934966 2024-12-06T16:02:19,130 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting fa940d57dea64247b64848ae358e02ac, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733500936151 2024-12-06T16:02:19,139 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81761a3d64f4403adedb17ba781753d2#C#compaction#350 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:19,139 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/5726587425d74704a7829dd5dec67235 is 50, key is test_row_0/C:col10/1733500936151/Put/seqid=0 2024-12-06T16:02:19,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742235_1411 (size=12409) 2024-12-06T16:02:19,156 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/5726587425d74704a7829dd5dec67235 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/5726587425d74704a7829dd5dec67235 2024-12-06T16:02:19,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-06T16:02:19,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:19,167 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 81761a3d64f4403adedb17ba781753d2 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-12-06T16:02:19,168 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81761a3d64f4403adedb17ba781753d2/C of 81761a3d64f4403adedb17ba781753d2 into 5726587425d74704a7829dd5dec67235(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:19,168 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:19,168 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., storeName=81761a3d64f4403adedb17ba781753d2/C, priority=13, startTime=1733500939090; duration=0sec 2024-12-06T16:02:19,169 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:19,169 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81761a3d64f4403adedb17ba781753d2:C 2024-12-06T16:02:19,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120658ee84c1fcbe439fac0998c5b4af07a8_81761a3d64f4403adedb17ba781753d2 is 50, key is test_row_0/A:col10/1733500937393/Put/seqid=0 2024-12-06T16:02:19,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742236_1412 (size=12304) 2024-12-06T16:02:19,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:02:19,195 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120658ee84c1fcbe439fac0998c5b4af07a8_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120658ee84c1fcbe439fac0998c5b4af07a8_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:19,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/d2860461e3b74111ad08dfb10419da05, store: [table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:19,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/d2860461e3b74111ad08dfb10419da05 is 175, key is test_row_0/A:col10/1733500937393/Put/seqid=0 2024-12-06T16:02:19,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742237_1413 (size=31105) 2024-12-06T16:02:19,205 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/d2860461e3b74111ad08dfb10419da05 2024-12-06T16:02:19,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/b98914dc6151451fb947e539de9266f0 is 50, key is test_row_0/B:col10/1733500937393/Put/seqid=0 2024-12-06T16:02:19,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742238_1414 (size=12151) 2024-12-06T16:02:19,525 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81761a3d64f4403adedb17ba781753d2#A#compaction#348 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:19,526 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/6d4b42ca541f4fc28e68a5d859365a5a is 175, key is test_row_0/A:col10/1733500936151/Put/seqid=0 2024-12-06T16:02:19,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742239_1415 (size=31363) 2024-12-06T16:02:19,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:19,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:19,622 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/b98914dc6151451fb947e539de9266f0 2024-12-06T16:02:19,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-06T16:02:19,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:19,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500999621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:19,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:19,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:19,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:19,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/299b042d850949dc873ea3e0a96439c8 is 50, key is test_row_0/C:col10/1733500937393/Put/seqid=0 2024-12-06T16:02:19,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742240_1416 (size=12151) 2024-12-06T16:02:19,634 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/299b042d850949dc873ea3e0a96439c8 2024-12-06T16:02:19,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/d2860461e3b74111ad08dfb10419da05 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/d2860461e3b74111ad08dfb10419da05 2024-12-06T16:02:19,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:19,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500999623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:19,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:19,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500999623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:19,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:19,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500999625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:19,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:19,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500999625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:19,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:19,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:19,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:19,643 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/d2860461e3b74111ad08dfb10419da05, entries=150, sequenceid=157, filesize=30.4 K 2024-12-06T16:02:19,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:19,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:19,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:19,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/b98914dc6151451fb947e539de9266f0 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/b98914dc6151451fb947e539de9266f0 2024-12-06T16:02:19,649 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/b98914dc6151451fb947e539de9266f0, entries=150, sequenceid=157, filesize=11.9 K 2024-12-06T16:02:19,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/299b042d850949dc873ea3e0a96439c8 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/299b042d850949dc873ea3e0a96439c8 2024-12-06T16:02:19,653 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/299b042d850949dc873ea3e0a96439c8, entries=150, sequenceid=157, filesize=11.9 K 2024-12-06T16:02:19,654 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 81761a3d64f4403adedb17ba781753d2 in 487ms, sequenceid=157, compaction requested=false 2024-12-06T16:02:19,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:19,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:19,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-12-06T16:02:19,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-12-06T16:02:19,657 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-06T16:02:19,657 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1790 sec 2024-12-06T16:02:19,659 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 2.1850 sec 2024-12-06T16:02:19,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:19,740 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:02:19,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81761a3d64f4403adedb17ba781753d2 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-12-06T16:02:19,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206bf7b190fb3be4bdbb7ec64e383b12b05_81761a3d64f4403adedb17ba781753d2 is 50, key is test_row_0/A:col10/1733500939621/Put/seqid=0 2024-12-06T16:02:19,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742241_1417 (size=14794) 2024-12-06T16:02:19,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:19,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500999780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:19,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:19,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:19,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:19,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:19,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500999790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:19,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:19,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:19,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:19,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:19,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500999791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:19,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:19,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500999792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:19,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:19,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:19,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:19,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:19,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500999792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:19,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:19,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:19,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:19,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:19,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:19,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:19,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:19,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733500999902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:19,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:19,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:19,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:19,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733500999902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:19,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:19,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:19,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:19,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:19,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:19,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733500999911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:19,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:19,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733500999912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:19,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:19,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:19,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:19,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:19,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:19,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:19,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:19,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733500999912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:19,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:19,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:19,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:19,935 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/6d4b42ca541f4fc28e68a5d859365a5a as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/6d4b42ca541f4fc28e68a5d859365a5a 2024-12-06T16:02:19,940 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81761a3d64f4403adedb17ba781753d2/A of 81761a3d64f4403adedb17ba781753d2 into 6d4b42ca541f4fc28e68a5d859365a5a(size=30.6 K), total size for store is 61.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:19,940 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:19,940 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., storeName=81761a3d64f4403adedb17ba781753d2/A, priority=13, startTime=1733500939090; duration=0sec 2024-12-06T16:02:19,940 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:19,940 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81761a3d64f4403adedb17ba781753d2:A 2024-12-06T16:02:20,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:20,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733501000112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:20,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:20,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733501000112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:20,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:20,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733501000131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:20,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:20,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733501000131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:20,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:20,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:20,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:20,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733501000131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:20,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:20,157 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:02:20,164 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206bf7b190fb3be4bdbb7ec64e383b12b05_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206bf7b190fb3be4bdbb7ec64e383b12b05_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:20,165 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/cc484b3cfc6b4d4a9e0ddc3afef04544, store: [table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:20,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/cc484b3cfc6b4d4a9e0ddc3afef04544 is 175, key is test_row_0/A:col10/1733500939621/Put/seqid=0 2024-12-06T16:02:20,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742242_1418 (size=39749) 2024-12-06T16:02:20,169 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/cc484b3cfc6b4d4a9e0ddc3afef04544 2024-12-06T16:02:20,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/0c55f3e579504521ab8416b76c4577c4 is 50, key is test_row_0/B:col10/1733500939621/Put/seqid=0 2024-12-06T16:02:20,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742243_1419 (size=12151) 2024-12-06T16:02:20,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/0c55f3e579504521ab8416b76c4577c4 2024-12-06T16:02:20,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/5b3dd7dc9e8244a28c946dc53404d2c3 is 50, key is test_row_0/C:col10/1733500939621/Put/seqid=0 2024-12-06T16:02:20,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742244_1420 (size=12151) 2024-12-06T16:02:20,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/5b3dd7dc9e8244a28c946dc53404d2c3 2024-12-06T16:02:20,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/cc484b3cfc6b4d4a9e0ddc3afef04544 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/cc484b3cfc6b4d4a9e0ddc3afef04544 2024-12-06T16:02:20,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/cc484b3cfc6b4d4a9e0ddc3afef04544, entries=200, sequenceid=174, filesize=38.8 K 2024-12-06T16:02:20,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/0c55f3e579504521ab8416b76c4577c4 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/0c55f3e579504521ab8416b76c4577c4 2024-12-06T16:02:20,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/0c55f3e579504521ab8416b76c4577c4, entries=150, sequenceid=174, filesize=11.9 K 2024-12-06T16:02:20,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/5b3dd7dc9e8244a28c946dc53404d2c3 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/5b3dd7dc9e8244a28c946dc53404d2c3 2024-12-06T16:02:20,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/5b3dd7dc9e8244a28c946dc53404d2c3, entries=150, sequenceid=174, filesize=11.9 K 2024-12-06T16:02:20,217 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 81761a3d64f4403adedb17ba781753d2 in 476ms, sequenceid=174, compaction requested=true 2024-12-06T16:02:20,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:20,217 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:20,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81761a3d64f4403adedb17ba781753d2:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:02:20,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:20,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81761a3d64f4403adedb17ba781753d2:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:02:20,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:20,217 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:20,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81761a3d64f4403adedb17ba781753d2:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:02:20,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:20,218 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:20,218 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 81761a3d64f4403adedb17ba781753d2/A is initiating minor compaction (all files) 2024-12-06T16:02:20,218 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81761a3d64f4403adedb17ba781753d2/A in TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:20,218 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/6d4b42ca541f4fc28e68a5d859365a5a, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/d2860461e3b74111ad08dfb10419da05, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/cc484b3cfc6b4d4a9e0ddc3afef04544] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp, totalSize=99.8 K 2024-12-06T16:02:20,218 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:20,218 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. files: [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/6d4b42ca541f4fc28e68a5d859365a5a, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/d2860461e3b74111ad08dfb10419da05, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/cc484b3cfc6b4d4a9e0ddc3afef04544] 2024-12-06T16:02:20,218 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:20,219 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 81761a3d64f4403adedb17ba781753d2/B is initiating minor compaction (all files) 2024-12-06T16:02:20,219 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81761a3d64f4403adedb17ba781753d2/B in TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:20,219 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/9df178cc4f3f4963871b88fc2e71591e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/b98914dc6151451fb947e539de9266f0, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/0c55f3e579504521ab8416b76c4577c4] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp, totalSize=35.9 K 2024-12-06T16:02:20,219 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d4b42ca541f4fc28e68a5d859365a5a, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733500936151 2024-12-06T16:02:20,219 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 9df178cc4f3f4963871b88fc2e71591e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733500936151 2024-12-06T16:02:20,219 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2860461e3b74111ad08dfb10419da05, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733500937367 2024-12-06T16:02:20,220 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc484b3cfc6b4d4a9e0ddc3afef04544, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733500939620 2024-12-06T16:02:20,220 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting b98914dc6151451fb947e539de9266f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733500937367 2024-12-06T16:02:20,220 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c55f3e579504521ab8416b76c4577c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733500939620 2024-12-06T16:02:20,230 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:20,231 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81761a3d64f4403adedb17ba781753d2#B#compaction#358 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:20,232 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/170110204f6c448e8c7700d36cc868bd is 50, key is test_row_0/B:col10/1733500939621/Put/seqid=0 2024-12-06T16:02:20,234 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120628fd6d2d66614639b5ff77f510aaca18_81761a3d64f4403adedb17ba781753d2 store=[table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:20,236 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120628fd6d2d66614639b5ff77f510aaca18_81761a3d64f4403adedb17ba781753d2, store=[table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:20,236 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120628fd6d2d66614639b5ff77f510aaca18_81761a3d64f4403adedb17ba781753d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:20,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742245_1421 (size=12561) 2024-12-06T16:02:20,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742246_1422 (size=4469) 2024-12-06T16:02:20,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:20,429 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:02:20,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81761a3d64f4403adedb17ba781753d2 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-12-06T16:02:20,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412067305347d67564b8b91b3954446e63608_81761a3d64f4403adedb17ba781753d2 is 50, key is test_row_0/A:col10/1733500939778/Put/seqid=0 2024-12-06T16:02:20,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742247_1423 (size=14794) 2024-12-06T16:02:20,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:20,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733501000466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:20,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733501000466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:20,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733501000470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:20,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:20,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733501000471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:20,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:20,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:20,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733501000472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:20,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:20,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:20,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733501000581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:20,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:20,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733501000581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:20,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:20,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:20,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:20,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733501000592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:20,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733501000592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:20,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733501000592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:20,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:20,655 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81761a3d64f4403adedb17ba781753d2#A#compaction#357 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:20,655 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/7eda3659f24f409291b58710d11c3e27 is 175, key is test_row_0/A:col10/1733500939621/Put/seqid=0 2024-12-06T16:02:20,656 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/170110204f6c448e8c7700d36cc868bd as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/170110204f6c448e8c7700d36cc868bd 2024-12-06T16:02:20,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742248_1424 (size=31515) 2024-12-06T16:02:20,661 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81761a3d64f4403adedb17ba781753d2/B of 81761a3d64f4403adedb17ba781753d2 into 170110204f6c448e8c7700d36cc868bd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:20,661 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:20,661 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., storeName=81761a3d64f4403adedb17ba781753d2/B, priority=13, startTime=1733500940217; duration=0sec 2024-12-06T16:02:20,661 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:20,661 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81761a3d64f4403adedb17ba781753d2:B 2024-12-06T16:02:20,661 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:20,662 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:20,662 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 81761a3d64f4403adedb17ba781753d2/C is initiating minor compaction (all files) 2024-12-06T16:02:20,663 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81761a3d64f4403adedb17ba781753d2/C in TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:20,663 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/5726587425d74704a7829dd5dec67235, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/299b042d850949dc873ea3e0a96439c8, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/5b3dd7dc9e8244a28c946dc53404d2c3] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp, totalSize=35.9 K 2024-12-06T16:02:20,663 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 5726587425d74704a7829dd5dec67235, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733500936151 2024-12-06T16:02:20,664 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 299b042d850949dc873ea3e0a96439c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733500937367 2024-12-06T16:02:20,665 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b3dd7dc9e8244a28c946dc53404d2c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733500939620 2024-12-06T16:02:20,671 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81761a3d64f4403adedb17ba781753d2#C#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:20,672 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/ce90db3619cc4529ba7f4faa1588eb84 is 50, key is test_row_0/C:col10/1733500939621/Put/seqid=0 2024-12-06T16:02:20,676 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/7eda3659f24f409291b58710d11c3e27 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/7eda3659f24f409291b58710d11c3e27 2024-12-06T16:02:20,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742249_1425 (size=12561) 2024-12-06T16:02:20,685 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/ce90db3619cc4529ba7f4faa1588eb84 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/ce90db3619cc4529ba7f4faa1588eb84 2024-12-06T16:02:20,685 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81761a3d64f4403adedb17ba781753d2/A of 81761a3d64f4403adedb17ba781753d2 into 7eda3659f24f409291b58710d11c3e27(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:20,685 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:20,685 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., storeName=81761a3d64f4403adedb17ba781753d2/A, priority=13, startTime=1733500940217; duration=0sec 2024-12-06T16:02:20,685 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:20,685 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81761a3d64f4403adedb17ba781753d2:A 2024-12-06T16:02:20,691 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81761a3d64f4403adedb17ba781753d2/C of 81761a3d64f4403adedb17ba781753d2 into ce90db3619cc4529ba7f4faa1588eb84(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:20,691 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:20,691 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., storeName=81761a3d64f4403adedb17ba781753d2/C, priority=13, startTime=1733500940217; duration=0sec 2024-12-06T16:02:20,691 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:20,691 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81761a3d64f4403adedb17ba781753d2:C 2024-12-06T16:02:20,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:20,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733501000792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:20,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:20,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733501000792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:20,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:20,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:20,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:20,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733501000811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:20,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:20,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733501000812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:20,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:20,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:20,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733501000812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:20,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:20,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:20,840 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:02:20,845 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412067305347d67564b8b91b3954446e63608_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412067305347d67564b8b91b3954446e63608_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:20,846 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/38cb136457634cefb2f4eb2f36cadfa5, store: [table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:20,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/38cb136457634cefb2f4eb2f36cadfa5 is 175, key is test_row_0/A:col10/1733500939778/Put/seqid=0 2024-12-06T16:02:20,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742250_1426 (size=39749) 2024-12-06T16:02:20,854 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/38cb136457634cefb2f4eb2f36cadfa5 2024-12-06T16:02:20,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/0171733e98fa400b9186f750e2868fd4 is 50, key is test_row_0/B:col10/1733500939778/Put/seqid=0 2024-12-06T16:02:20,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742251_1427 (size=12151) 2024-12-06T16:02:20,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/0171733e98fa400b9186f750e2868fd4 2024-12-06T16:02:20,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/da1807f35a8647238c4f7e27254f5a7d is 50, key is test_row_0/C:col10/1733500939778/Put/seqid=0 2024-12-06T16:02:20,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742252_1428 (size=12151) 2024-12-06T16:02:21,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:21,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733501001112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:21,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:21,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:21,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:21,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733501001112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:21,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:21,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:21,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:21,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:21,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733501001132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:21,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:21,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:21,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733501001132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:21,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:21,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:21,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:21,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733501001132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:21,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:21,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:21,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:21,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:21,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:21,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:21,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/da1807f35a8647238c4f7e27254f5a7d 2024-12-06T16:02:21,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/38cb136457634cefb2f4eb2f36cadfa5 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/38cb136457634cefb2f4eb2f36cadfa5 2024-12-06T16:02:21,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/38cb136457634cefb2f4eb2f36cadfa5, entries=200, sequenceid=195, filesize=38.8 K 2024-12-06T16:02:21,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/0171733e98fa400b9186f750e2868fd4 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/0171733e98fa400b9186f750e2868fd4 2024-12-06T16:02:21,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/0171733e98fa400b9186f750e2868fd4, entries=150, sequenceid=195, filesize=11.9 K 2024-12-06T16:02:21,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/da1807f35a8647238c4f7e27254f5a7d as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/da1807f35a8647238c4f7e27254f5a7d 2024-12-06T16:02:21,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/da1807f35a8647238c4f7e27254f5a7d, entries=150, sequenceid=195, filesize=11.9 K 2024-12-06T16:02:21,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 81761a3d64f4403adedb17ba781753d2 in 867ms, sequenceid=195, compaction requested=false 2024-12-06T16:02:21,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:21,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:21,638 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:02:21,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81761a3d64f4403adedb17ba781753d2 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-12-06T16:02:21,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-06T16:02:21,642 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:02:21,642 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:02:21,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-12-06T16:02:21,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-06T16:02:21,644 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:02:21,645 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:02:21,645 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:02:21,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412062f0c898f98074a138803554b77e252d1_81761a3d64f4403adedb17ba781753d2 is 50, key is test_row_0/A:col10/1733500941636/Put/seqid=0 2024-12-06T16:02:21,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742253_1429 (size=14794) 2024-12-06T16:02:21,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:21,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733501001679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:21,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:21,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733501001681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:21,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:21,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733501001682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:21,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:21,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:21,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:21,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:21,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733501001694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:21,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:21,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:21,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:21,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:21,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:21,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:21,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:21,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:21,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:21,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:21,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733501001697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:21,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:21,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:21,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:21,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-06T16:02:21,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-06T16:02:21,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:21,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:21,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:21,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:21,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:21,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:21,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:21,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733501001801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:21,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:21,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733501001801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:21,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:21,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733501001801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:21,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:21,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:21,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733501001801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:21,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:21,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:21,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:21,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:21,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733501001822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:21,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:21,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:21,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:21,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-06T16:02:21,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:21,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:21,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:21,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:21,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-06T16:02:22,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733501002011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:22,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:22,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:22,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:22,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733501002012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:22,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:22,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:22,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:22,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733501002022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:22,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:22,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:22,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:22,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733501002022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:22,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:22,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:22,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:22,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:22,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733501002041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:22,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:22,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:22,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:22,055 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:02:22,058 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412062f0c898f98074a138803554b77e252d1_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412062f0c898f98074a138803554b77e252d1_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:22,059 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/06c9fbb162d44ca6a188a79b07aacfd6, store: [table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:22,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/06c9fbb162d44ca6a188a79b07aacfd6 is 175, key is test_row_0/A:col10/1733500941636/Put/seqid=0 2024-12-06T16:02:22,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742254_1430 (size=39749) 2024-12-06T16:02:22,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-06T16:02:22,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:22,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:22,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:22,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:22,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:22,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-06T16:02:22,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:22,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:22,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:22,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:22,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:22,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:22,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-06T16:02:22,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733501002332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:22,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:22,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:22,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733501002332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:22,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:22,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733501002332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:22,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:22,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:22,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:22,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:22,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:22,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:22,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733501002341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:22,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:22,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:22,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:22,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733501002362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:22,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:22,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:22,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:22,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-06T16:02:22,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:22,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:22,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:22,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:22,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:22,464 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/06c9fbb162d44ca6a188a79b07aacfd6 2024-12-06T16:02:22,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/548f4f78c1a1415087cef8823dddafa3 is 50, key is test_row_0/B:col10/1733500941636/Put/seqid=0 2024-12-06T16:02:22,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742255_1431 (size=12151) 2024-12-06T16:02:22,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-06T16:02:22,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:22,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:22,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:22,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:22,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:22,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-06T16:02:22,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:22,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:22,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:22,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:22,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-06T16:02:22,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733501002851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:22,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:22,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:22,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733501002852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:22,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:22,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:22,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:22,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:22,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733501002852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:22,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:22,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:22,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:22,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733501002852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:22,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:22,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:22,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:22,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-06T16:02:22,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:22,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:22,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:22,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:22,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:22,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/548f4f78c1a1415087cef8823dddafa3 2024-12-06T16:02:22,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/63eb4bb05c5a40778eb8fb2744db461e is 50, key is test_row_0/C:col10/1733500941636/Put/seqid=0 2024-12-06T16:02:22,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733501002882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:22,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:22,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:22,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:22,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742256_1432 (size=12151) 2024-12-06T16:02:23,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-06T16:02:23,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:23,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:23,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:23,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:23,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:23,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-06T16:02:23,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:23,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:23,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:23,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:23,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:23,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/63eb4bb05c5a40778eb8fb2744db461e 2024-12-06T16:02:23,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/06c9fbb162d44ca6a188a79b07aacfd6 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/06c9fbb162d44ca6a188a79b07aacfd6 2024-12-06T16:02:23,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/06c9fbb162d44ca6a188a79b07aacfd6, entries=200, sequenceid=216, filesize=38.8 K 2024-12-06T16:02:23,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/548f4f78c1a1415087cef8823dddafa3 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/548f4f78c1a1415087cef8823dddafa3 2024-12-06T16:02:23,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/548f4f78c1a1415087cef8823dddafa3, entries=150, sequenceid=216, filesize=11.9 K 2024-12-06T16:02:23,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/63eb4bb05c5a40778eb8fb2744db461e as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/63eb4bb05c5a40778eb8fb2744db461e 2024-12-06T16:02:23,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/63eb4bb05c5a40778eb8fb2744db461e, entries=150, sequenceid=216, filesize=11.9 K 2024-12-06T16:02:23,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 81761a3d64f4403adedb17ba781753d2 in 1672ms, sequenceid=216, compaction requested=true 2024-12-06T16:02:23,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:23,310 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:23,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81761a3d64f4403adedb17ba781753d2:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:02:23,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:23,310 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:23,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81761a3d64f4403adedb17ba781753d2:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:02:23,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:23,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81761a3d64f4403adedb17ba781753d2:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:02:23,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:23,311 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:23,311 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 81761a3d64f4403adedb17ba781753d2/A is initiating minor compaction (all files) 2024-12-06T16:02:23,311 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81761a3d64f4403adedb17ba781753d2/A in TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:23,311 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/7eda3659f24f409291b58710d11c3e27, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/38cb136457634cefb2f4eb2f36cadfa5, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/06c9fbb162d44ca6a188a79b07aacfd6] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp, totalSize=108.4 K 2024-12-06T16:02:23,311 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:23,311 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. files: [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/7eda3659f24f409291b58710d11c3e27, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/38cb136457634cefb2f4eb2f36cadfa5, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/06c9fbb162d44ca6a188a79b07aacfd6] 2024-12-06T16:02:23,312 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:23,312 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 81761a3d64f4403adedb17ba781753d2/B is initiating minor compaction (all files) 2024-12-06T16:02:23,312 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81761a3d64f4403adedb17ba781753d2/B in TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:23,312 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/170110204f6c448e8c7700d36cc868bd, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/0171733e98fa400b9186f750e2868fd4, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/548f4f78c1a1415087cef8823dddafa3] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp, totalSize=36.0 K 2024-12-06T16:02:23,312 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7eda3659f24f409291b58710d11c3e27, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733500939620 2024-12-06T16:02:23,312 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 170110204f6c448e8c7700d36cc868bd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733500939620 2024-12-06T16:02:23,312 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38cb136457634cefb2f4eb2f36cadfa5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1733500939778 2024-12-06T16:02:23,313 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 0171733e98fa400b9186f750e2868fd4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1733500939778 2024-12-06T16:02:23,313 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06c9fbb162d44ca6a188a79b07aacfd6, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733500940470 2024-12-06T16:02:23,313 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 548f4f78c1a1415087cef8823dddafa3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733500940471 2024-12-06T16:02:23,321 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:23,328 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81761a3d64f4403adedb17ba781753d2#B#compaction#367 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:23,328 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/3af9e327dc024a38adc185d52999b960 is 50, key is test_row_0/B:col10/1733500941636/Put/seqid=0 2024-12-06T16:02:23,330 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120654db88a14c824900ac9802a0c14ae305_81761a3d64f4403adedb17ba781753d2 store=[table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:23,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-06T16:02:23,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:23,330 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 81761a3d64f4403adedb17ba781753d2 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-12-06T16:02:23,332 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120654db88a14c824900ac9802a0c14ae305_81761a3d64f4403adedb17ba781753d2, store=[table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:23,332 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120654db88a14c824900ac9802a0c14ae305_81761a3d64f4403adedb17ba781753d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:23,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742257_1433 (size=12663) 2024-12-06T16:02:23,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206289ba9b0398a4dd7856fc4d47b40f714_81761a3d64f4403adedb17ba781753d2 is 50, key is test_row_0/A:col10/1733500941680/Put/seqid=0 2024-12-06T16:02:23,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742258_1434 (size=4469) 2024-12-06T16:02:23,350 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81761a3d64f4403adedb17ba781753d2#A#compaction#366 average throughput is 0.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:23,351 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/a16d83b3705a4667a8e6516527417926 is 175, key is test_row_0/A:col10/1733500941636/Put/seqid=0 2024-12-06T16:02:23,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742260_1436 (size=31617) 2024-12-06T16:02:23,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742259_1435 (size=12304) 2024-12-06T16:02:23,372 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/a16d83b3705a4667a8e6516527417926 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/a16d83b3705a4667a8e6516527417926 2024-12-06T16:02:23,378 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81761a3d64f4403adedb17ba781753d2/A of 81761a3d64f4403adedb17ba781753d2 into a16d83b3705a4667a8e6516527417926(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:23,378 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:23,378 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., storeName=81761a3d64f4403adedb17ba781753d2/A, priority=13, startTime=1733500943310; duration=0sec 2024-12-06T16:02:23,378 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:23,378 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81761a3d64f4403adedb17ba781753d2:A 2024-12-06T16:02:23,378 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:23,379 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:23,380 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 81761a3d64f4403adedb17ba781753d2/C is initiating minor compaction (all files) 2024-12-06T16:02:23,380 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81761a3d64f4403adedb17ba781753d2/C in TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:23,380 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/ce90db3619cc4529ba7f4faa1588eb84, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/da1807f35a8647238c4f7e27254f5a7d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/63eb4bb05c5a40778eb8fb2744db461e] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp, totalSize=36.0 K 2024-12-06T16:02:23,380 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce90db3619cc4529ba7f4faa1588eb84, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733500939620 2024-12-06T16:02:23,381 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting da1807f35a8647238c4f7e27254f5a7d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1733500939778 2024-12-06T16:02:23,381 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63eb4bb05c5a40778eb8fb2744db461e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733500940471 2024-12-06T16:02:23,393 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81761a3d64f4403adedb17ba781753d2#C#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:23,394 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/f1575e3b55004dc5a5b6662993c41142 is 50, key is test_row_0/C:col10/1733500941636/Put/seqid=0 2024-12-06T16:02:23,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742261_1437 (size=12663) 2024-12-06T16:02:23,407 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/f1575e3b55004dc5a5b6662993c41142 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/f1575e3b55004dc5a5b6662993c41142 2024-12-06T16:02:23,411 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81761a3d64f4403adedb17ba781753d2/C of 81761a3d64f4403adedb17ba781753d2 into f1575e3b55004dc5a5b6662993c41142(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:23,411 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:23,411 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., storeName=81761a3d64f4403adedb17ba781753d2/C, priority=13, startTime=1733500943311; duration=0sec 2024-12-06T16:02:23,411 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:23,411 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81761a3d64f4403adedb17ba781753d2:C 2024-12-06T16:02:23,743 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/3af9e327dc024a38adc185d52999b960 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/3af9e327dc024a38adc185d52999b960 2024-12-06T16:02:23,749 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81761a3d64f4403adedb17ba781753d2/B of 81761a3d64f4403adedb17ba781753d2 into 3af9e327dc024a38adc185d52999b960(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:23,749 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:23,749 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., storeName=81761a3d64f4403adedb17ba781753d2/B, priority=13, startTime=1733500943310; duration=0sec 2024-12-06T16:02:23,749 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:23,749 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81761a3d64f4403adedb17ba781753d2:B 2024-12-06T16:02:23,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:02:23,769 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206289ba9b0398a4dd7856fc4d47b40f714_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206289ba9b0398a4dd7856fc4d47b40f714_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:23,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/db95458be79d45f6808eac29ed79fa14, store: [table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:23,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/db95458be79d45f6808eac29ed79fa14 is 175, key is test_row_0/A:col10/1733500941680/Put/seqid=0 2024-12-06T16:02:23,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742262_1438 (size=31105) 2024-12-06T16:02:23,775 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/db95458be79d45f6808eac29ed79fa14 2024-12-06T16:02:23,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/3f14c595f76a4a0ebdbfaeee5d4d44ba is 50, key is test_row_0/B:col10/1733500941680/Put/seqid=0 2024-12-06T16:02:23,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742263_1439 (size=12151) 2024-12-06T16:02:23,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-06T16:02:23,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:23,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:23,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:23,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733501003907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:23,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:23,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733501003907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:23,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:23,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:23,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:23,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:23,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:23,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:23,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:23,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733501003908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:23,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:23,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:23,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:23,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:23,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733501003911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:23,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:23,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:23,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:23,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:23,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733501003912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:23,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:23,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:23,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:24,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:24,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733501004022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:24,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733501004022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:24,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733501004032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:24,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733501004032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:24,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:24,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:24,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:24,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:24,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:24,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733501004032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:24,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:24,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:24,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:24,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:24,189 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/3f14c595f76a4a0ebdbfaeee5d4d44ba 2024-12-06T16:02:24,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/ea3b71a81dd94fcd8f5539e8b079efda is 50, key is test_row_0/C:col10/1733500941680/Put/seqid=0 2024-12-06T16:02:24,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742264_1440 (size=12151) 2024-12-06T16:02:24,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:24,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733501004232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:24,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:24,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:24,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:24,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:24,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733501004242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:24,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:24,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:24,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:24,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:24,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733501004251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:24,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:24,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:24,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:24,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733501004251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:24,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:24,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:24,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733501004252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:24,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:24,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:24,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:24,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:24,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:24,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:24,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:24,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733501004552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:24,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:24,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:24,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:24,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733501004562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:24,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:24,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:24,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733501004562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:24,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733501004562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:24,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733501004572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:24,602 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/ea3b71a81dd94fcd8f5539e8b079efda 2024-12-06T16:02:24,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/db95458be79d45f6808eac29ed79fa14 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/db95458be79d45f6808eac29ed79fa14 2024-12-06T16:02:24,611 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/db95458be79d45f6808eac29ed79fa14, entries=150, sequenceid=234, filesize=30.4 K 2024-12-06T16:02:24,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/3f14c595f76a4a0ebdbfaeee5d4d44ba as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/3f14c595f76a4a0ebdbfaeee5d4d44ba 2024-12-06T16:02:24,618 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/3f14c595f76a4a0ebdbfaeee5d4d44ba, entries=150, sequenceid=234, filesize=11.9 K 2024-12-06T16:02:24,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/ea3b71a81dd94fcd8f5539e8b079efda as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/ea3b71a81dd94fcd8f5539e8b079efda 2024-12-06T16:02:24,623 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/ea3b71a81dd94fcd8f5539e8b079efda, entries=150, sequenceid=234, filesize=11.9 K 2024-12-06T16:02:24,624 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 81761a3d64f4403adedb17ba781753d2 in 1294ms, sequenceid=234, compaction requested=false 2024-12-06T16:02:24,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:24,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:24,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-12-06T16:02:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-12-06T16:02:24,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-06T16:02:24,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9830 sec 2024-12-06T16:02:24,632 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.9880 sec 2024-12-06T16:02:25,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:25,069 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:02:25,069 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81761a3d64f4403adedb17ba781753d2 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-12-06T16:02:25,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120678de496b507f499990e5013099106f82_81761a3d64f4403adedb17ba781753d2 is 50, key is test_row_0/A:col10/1733500943904/Put/seqid=0 2024-12-06T16:02:25,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742265_1441 (size=14794) 2024-12-06T16:02:25,091 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:02:25,097 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120678de496b507f499990e5013099106f82_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120678de496b507f499990e5013099106f82_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:25,098 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/5a8a531e0a5c4112b4d7913ae1063af8, store: [table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:25,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/5a8a531e0a5c4112b4d7913ae1063af8 is 175, key is test_row_0/A:col10/1733500943904/Put/seqid=0 2024-12-06T16:02:25,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742266_1442 (size=39749) 2024-12-06T16:02:25,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:25,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733501005109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:25,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:25,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733501005109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:25,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:25,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:25,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:25,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733501005110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:25,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:25,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:25,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733501005111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:25,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:25,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733501005113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:25,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:25,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:25,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:25,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733501005232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:25,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:25,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733501005232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:25,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:25,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:25,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733501005232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:25,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:25,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:25,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733501005232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:25,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:25,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:25,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:25,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733501005241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:25,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:25,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:25,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733501005452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:25,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:25,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733501005452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:25,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:25,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:25,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733501005452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:25,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:25,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:25,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733501005452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:25,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:25,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:25,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:25,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733501005462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:25,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:25,508 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=256, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/5a8a531e0a5c4112b4d7913ae1063af8 2024-12-06T16:02:25,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/489c176aceba4cd391684f482ef8708c is 50, key is test_row_0/B:col10/1733500943904/Put/seqid=0 2024-12-06T16:02:25,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742267_1443 (size=12151) 2024-12-06T16:02:25,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/489c176aceba4cd391684f482ef8708c 2024-12-06T16:02:25,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/5ef7e58fc1f3412e9a8c8541b126f815 is 50, key is test_row_0/C:col10/1733500943904/Put/seqid=0 2024-12-06T16:02:25,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742268_1444 (size=12151) 2024-12-06T16:02:25,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733501005772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:25,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733501005772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:25,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733501005772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:25,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733501005772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:25,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:25,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:25,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733501005772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:25,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:25,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:25,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-06T16:02:25,812 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:02:25,813 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:02:25,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-06T16:02:25,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-06T16:02:25,815 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:02:25,815 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:02:25,815 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:02:25,827 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-06T16:02:25,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-06T16:02:25,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/5ef7e58fc1f3412e9a8c8541b126f815 2024-12-06T16:02:25,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/5a8a531e0a5c4112b4d7913ae1063af8 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/5a8a531e0a5c4112b4d7913ae1063af8 2024-12-06T16:02:25,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/5a8a531e0a5c4112b4d7913ae1063af8, entries=200, sequenceid=256, filesize=38.8 K 2024-12-06T16:02:25,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/489c176aceba4cd391684f482ef8708c as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/489c176aceba4cd391684f482ef8708c 2024-12-06T16:02:25,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/489c176aceba4cd391684f482ef8708c, entries=150, sequenceid=256, filesize=11.9 K 2024-12-06T16:02:25,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/5ef7e58fc1f3412e9a8c8541b126f815 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/5ef7e58fc1f3412e9a8c8541b126f815 2024-12-06T16:02:25,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/5ef7e58fc1f3412e9a8c8541b126f815, entries=150, sequenceid=256, filesize=11.9 K 2024-12-06T16:02:25,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 81761a3d64f4403adedb17ba781753d2 in 882ms, sequenceid=256, compaction requested=true 2024-12-06T16:02:25,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:25,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81761a3d64f4403adedb17ba781753d2:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:02:25,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:25,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81761a3d64f4403adedb17ba781753d2:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:02:25,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:25,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81761a3d64f4403adedb17ba781753d2:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:02:25,951 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:25,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:25,951 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:25,952 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:25,952 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 81761a3d64f4403adedb17ba781753d2/A is initiating minor compaction (all files) 2024-12-06T16:02:25,952 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:25,952 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81761a3d64f4403adedb17ba781753d2/A in TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:25,952 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 81761a3d64f4403adedb17ba781753d2/B is initiating minor compaction (all files) 2024-12-06T16:02:25,952 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/a16d83b3705a4667a8e6516527417926, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/db95458be79d45f6808eac29ed79fa14, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/5a8a531e0a5c4112b4d7913ae1063af8] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp, totalSize=100.1 K 2024-12-06T16:02:25,952 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81761a3d64f4403adedb17ba781753d2/B in TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:25,952 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:25,953 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/3af9e327dc024a38adc185d52999b960, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/3f14c595f76a4a0ebdbfaeee5d4d44ba, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/489c176aceba4cd391684f482ef8708c] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp, totalSize=36.1 K 2024-12-06T16:02:25,953 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. files: [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/a16d83b3705a4667a8e6516527417926, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/db95458be79d45f6808eac29ed79fa14, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/5a8a531e0a5c4112b4d7913ae1063af8] 2024-12-06T16:02:25,953 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting a16d83b3705a4667a8e6516527417926, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733500940471 2024-12-06T16:02:25,953 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 3af9e327dc024a38adc185d52999b960, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733500940471 2024-12-06T16:02:25,953 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f14c595f76a4a0ebdbfaeee5d4d44ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733500941680 2024-12-06T16:02:25,953 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting db95458be79d45f6808eac29ed79fa14, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733500941680 2024-12-06T16:02:25,953 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 489c176aceba4cd391684f482ef8708c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733500943904 2024-12-06T16:02:25,953 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a8a531e0a5c4112b4d7913ae1063af8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733500943904 2024-12-06T16:02:25,959 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:25,961 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120675e15b7e7596426da5735a28b9b922f1_81761a3d64f4403adedb17ba781753d2 store=[table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:25,961 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81761a3d64f4403adedb17ba781753d2#B#compaction#376 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:25,962 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/8b624e0654694974987ca8a9b03a8cf1 is 50, key is test_row_0/B:col10/1733500943904/Put/seqid=0 2024-12-06T16:02:25,963 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120675e15b7e7596426da5735a28b9b922f1_81761a3d64f4403adedb17ba781753d2, store=[table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:25,964 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120675e15b7e7596426da5735a28b9b922f1_81761a3d64f4403adedb17ba781753d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:25,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-06T16:02:25,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:25,967 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 81761a3d64f4403adedb17ba781753d2 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-12-06T16:02:25,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742269_1445 (size=12765) 2024-12-06T16:02:25,980 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/8b624e0654694974987ca8a9b03a8cf1 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/8b624e0654694974987ca8a9b03a8cf1 2024-12-06T16:02:25,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742270_1446 (size=4469) 2024-12-06T16:02:25,995 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81761a3d64f4403adedb17ba781753d2#A#compaction#375 average throughput is 0.68 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:25,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206a16ba1e2087b410fb6c40191badf2bf4_81761a3d64f4403adedb17ba781753d2 is 50, key is test_row_0/A:col10/1733500945111/Put/seqid=0 2024-12-06T16:02:25,996 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81761a3d64f4403adedb17ba781753d2/B of 81761a3d64f4403adedb17ba781753d2 into 8b624e0654694974987ca8a9b03a8cf1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:25,996 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:25,996 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., storeName=81761a3d64f4403adedb17ba781753d2/B, priority=13, startTime=1733500945951; duration=0sec 2024-12-06T16:02:25,996 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:25,996 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81761a3d64f4403adedb17ba781753d2:B 2024-12-06T16:02:25,996 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:25,996 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/15409cdf862c4c1c818630021b594bc4 is 175, key is test_row_0/A:col10/1733500943904/Put/seqid=0 2024-12-06T16:02:25,997 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:25,997 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 81761a3d64f4403adedb17ba781753d2/C is initiating minor compaction (all files) 2024-12-06T16:02:25,997 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81761a3d64f4403adedb17ba781753d2/C in TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:25,997 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/f1575e3b55004dc5a5b6662993c41142, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/ea3b71a81dd94fcd8f5539e8b079efda, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/5ef7e58fc1f3412e9a8c8541b126f815] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp, totalSize=36.1 K 2024-12-06T16:02:26,000 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting f1575e3b55004dc5a5b6662993c41142, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733500940471 2024-12-06T16:02:26,000 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting ea3b71a81dd94fcd8f5539e8b079efda, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733500941680 2024-12-06T16:02:26,001 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ef7e58fc1f3412e9a8c8541b126f815, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733500943904 2024-12-06T16:02:26,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742271_1447 (size=12454) 2024-12-06T16:02:26,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:02:26,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742272_1448 (size=31719) 2024-12-06T16:02:26,009 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206a16ba1e2087b410fb6c40191badf2bf4_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a16ba1e2087b410fb6c40191badf2bf4_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:26,010 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81761a3d64f4403adedb17ba781753d2#C#compaction#378 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:26,010 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/c2b4b4f4bcd245648a9a3ad0e88f10e3 is 50, key is test_row_0/C:col10/1733500943904/Put/seqid=0 2024-12-06T16:02:26,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/5a825039295e475f9c490d963431fe95, store: [table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:26,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/5a825039295e475f9c490d963431fe95 is 175, key is test_row_0/A:col10/1733500945111/Put/seqid=0 2024-12-06T16:02:26,015 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/15409cdf862c4c1c818630021b594bc4 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/15409cdf862c4c1c818630021b594bc4 2024-12-06T16:02:26,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742273_1449 (size=12765) 2024-12-06T16:02:26,021 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81761a3d64f4403adedb17ba781753d2/A of 81761a3d64f4403adedb17ba781753d2 into 15409cdf862c4c1c818630021b594bc4(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:26,021 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:26,021 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., storeName=81761a3d64f4403adedb17ba781753d2/A, priority=13, startTime=1733500945951; duration=0sec 2024-12-06T16:02:26,021 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:26,021 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81761a3d64f4403adedb17ba781753d2:A 2024-12-06T16:02:26,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742274_1450 (size=31255) 2024-12-06T16:02:26,026 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/c2b4b4f4bcd245648a9a3ad0e88f10e3 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/c2b4b4f4bcd245648a9a3ad0e88f10e3 2024-12-06T16:02:26,034 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81761a3d64f4403adedb17ba781753d2/C of 81761a3d64f4403adedb17ba781753d2 into c2b4b4f4bcd245648a9a3ad0e88f10e3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:26,034 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:26,034 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., storeName=81761a3d64f4403adedb17ba781753d2/C, priority=13, startTime=1733500945951; duration=0sec 2024-12-06T16:02:26,034 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:26,034 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81761a3d64f4403adedb17ba781753d2:C 2024-12-06T16:02:26,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-06T16:02:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:26,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:26,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733501006315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:26,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:26,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733501006315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:26,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:26,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:26,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:26,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:26,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:26,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733501006316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:26,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:26,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:26,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:26,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733501006325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:26,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:26,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:26,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:26,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733501006326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:26,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:26,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:26,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:26,427 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/5a825039295e475f9c490d963431fe95 2024-12-06T16:02:26,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/cd8e12fc146e450a94ea1f530a154287 is 50, key is test_row_0/B:col10/1733500945111/Put/seqid=0 2024-12-06T16:02:26,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742275_1451 (size=12301) 2024-12-06T16:02:26,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-06T16:02:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733501006432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:26,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733501006432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:26,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:26,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:26,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:26,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733501006432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:26,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:26,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:26,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:26,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733501006432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:26,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:26,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:26,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:26,444 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/cd8e12fc146e450a94ea1f530a154287 2024-12-06T16:02:26,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:26,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:26,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:26,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/db623fb025cd4576a7a4bbb9f082b9c8 is 50, key is test_row_0/C:col10/1733500945111/Put/seqid=0 2024-12-06T16:02:26,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733501006444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:26,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:26,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:26,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:26,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742276_1452 (size=12301) 2024-12-06T16:02:26,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733501006652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:26,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:26,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:26,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:26,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733501006652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:26,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:26,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733501006653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:26,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:26,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:26,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:26,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:26,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:26,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733501006653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:26,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:26,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:26,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:26,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733501006662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:26,858 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/db623fb025cd4576a7a4bbb9f082b9c8 2024-12-06T16:02:26,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/5a825039295e475f9c490d963431fe95 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/5a825039295e475f9c490d963431fe95 2024-12-06T16:02:26,867 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/5a825039295e475f9c490d963431fe95, entries=150, sequenceid=273, filesize=30.5 K 2024-12-06T16:02:26,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/cd8e12fc146e450a94ea1f530a154287 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/cd8e12fc146e450a94ea1f530a154287 2024-12-06T16:02:26,873 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/cd8e12fc146e450a94ea1f530a154287, entries=150, sequenceid=273, filesize=12.0 K 2024-12-06T16:02:26,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/db623fb025cd4576a7a4bbb9f082b9c8 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/db623fb025cd4576a7a4bbb9f082b9c8 2024-12-06T16:02:26,877 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/db623fb025cd4576a7a4bbb9f082b9c8, entries=150, sequenceid=273, filesize=12.0 K 2024-12-06T16:02:26,878 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 81761a3d64f4403adedb17ba781753d2 in 911ms, sequenceid=273, compaction requested=false 2024-12-06T16:02:26,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:26,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:26,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-06T16:02:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-06T16:02:26,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-06T16:02:26,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0660 sec 2024-12-06T16:02:26,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.0700 sec 2024-12-06T16:02:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-06T16:02:26,952 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:02:26,953 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:02:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-06T16:02:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-06T16:02:26,955 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:02:26,956 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:02:26,956 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:02:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:26,978 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:02:26,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81761a3d64f4403adedb17ba781753d2 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-12-06T16:02:26,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206c4f21a94291142829502f610990b27f5_81761a3d64f4403adedb17ba781753d2 is 50, key is test_row_0/A:col10/1733500946976/Put/seqid=0 2024-12-06T16:02:26,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742277_1453 (size=14994) 2024-12-06T16:02:27,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:27,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733501006999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:27,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:27,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733501007011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:27,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:27,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:27,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733501007012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:27,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:27,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733501007013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:27,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:27,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733501007014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:27,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-06T16:02:27,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-06T16:02:27,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:27,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:27,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:27,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:27,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:27,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733501007121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:27,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:27,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733501007122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:27,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:27,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:27,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733501007132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:27,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:27,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733501007132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:27,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:27,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733501007132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:27,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:27,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-06T16:02:27,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:27,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:27,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:27,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:27,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:27,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:27,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-06T16:02:27,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:27,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733501007342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:27,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:27,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733501007342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:27,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:27,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:27,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:27,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733501007342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:27,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:27,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:27,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733501007342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:27,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:27,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:27,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733501007343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:27,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:27,390 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:02:27,394 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206c4f21a94291142829502f610990b27f5_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206c4f21a94291142829502f610990b27f5_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:27,395 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/57153e3385324478a445b86c7bd76a41, store: [table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:27,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/57153e3385324478a445b86c7bd76a41 is 175, key is test_row_0/A:col10/1733500946976/Put/seqid=0 2024-12-06T16:02:27,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742278_1454 (size=39949) 2024-12-06T16:02:27,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-06T16:02:27,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:27,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:27,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:27,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:27,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:27,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:27,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-06T16:02:27,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:27,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:27,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:27,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:27,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:27,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:27,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-06T16:02:27,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733501007662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:27,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733501007662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:27,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:27,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:27,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733501007663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:27,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:27,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733501007663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:27,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:27,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:27,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:27,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733501007663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:27,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:27,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:27,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-06T16:02:27,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:27,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:27,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:27,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:27,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:27,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:27,800 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=296, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/57153e3385324478a445b86c7bd76a41 2024-12-06T16:02:27,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/622016923b17458196436099dc1d1fa3 is 50, key is test_row_0/B:col10/1733500946976/Put/seqid=0 2024-12-06T16:02:27,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742279_1455 (size=12301) 2024-12-06T16:02:27,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-06T16:02:27,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:27,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:27,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:27,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:27,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:27,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:28,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-06T16:02:28,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:28,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:28,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:28,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:28,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:28,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:28,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-06T16:02:28,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-06T16:02:28,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:28,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:28,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:28,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:28,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:28,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:28,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:28,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35912 deadline: 1733501008182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:28,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:28,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:28,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:28,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:28,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35892 deadline: 1733501008182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:28,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:28,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:28,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:28,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:28,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35890 deadline: 1733501008183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:28,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:28,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:28,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:28,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:28,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1733501008183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:28,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:28,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:28,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:28,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:28,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35932 deadline: 1733501008183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:28,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:28,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:28,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:02:28,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/622016923b17458196436099dc1d1fa3 2024-12-06T16:02:28,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/781d7663cd574043a6e58a803c70b123 is 50, key is test_row_0/C:col10/1733500946976/Put/seqid=0 2024-12-06T16:02:28,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742280_1456 (size=12301) 2024-12-06T16:02:28,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-06T16:02:28,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:28,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:28,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:28,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:28,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:28,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-06T16:02:28,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:28,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:28,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:28,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:28,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:28,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/781d7663cd574043a6e58a803c70b123 2024-12-06T16:02:28,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/57153e3385324478a445b86c7bd76a41 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/57153e3385324478a445b86c7bd76a41 2024-12-06T16:02:28,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/57153e3385324478a445b86c7bd76a41, entries=200, sequenceid=296, filesize=39.0 K 2024-12-06T16:02:28,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/622016923b17458196436099dc1d1fa3 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/622016923b17458196436099dc1d1fa3 2024-12-06T16:02:28,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-06T16:02:28,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:28,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:28,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:28,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:28,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:28,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/622016923b17458196436099dc1d1fa3, entries=150, sequenceid=296, filesize=12.0 K 2024-12-06T16:02:28,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/781d7663cd574043a6e58a803c70b123 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/781d7663cd574043a6e58a803c70b123 2024-12-06T16:02:28,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/781d7663cd574043a6e58a803c70b123, entries=150, sequenceid=296, filesize=12.0 K 2024-12-06T16:02:28,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 81761a3d64f4403adedb17ba781753d2 in 1678ms, sequenceid=296, compaction requested=true 2024-12-06T16:02:28,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:28,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81761a3d64f4403adedb17ba781753d2:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:02:28,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:28,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81761a3d64f4403adedb17ba781753d2:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:02:28,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-06T16:02:28,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81761a3d64f4403adedb17ba781753d2:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:02:28,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-06T16:02:28,656 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:28,657 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:28,658 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:28,658 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 81761a3d64f4403adedb17ba781753d2/C is initiating minor compaction (all files) 2024-12-06T16:02:28,658 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81761a3d64f4403adedb17ba781753d2/C in TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:28,658 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/c2b4b4f4bcd245648a9a3ad0e88f10e3, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/db623fb025cd4576a7a4bbb9f082b9c8, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/781d7663cd574043a6e58a803c70b123] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp, totalSize=36.5 K 2024-12-06T16:02:28,658 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:28,658 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): 81761a3d64f4403adedb17ba781753d2/A is initiating minor compaction (all files) 2024-12-06T16:02:28,658 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81761a3d64f4403adedb17ba781753d2/A in TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:28,658 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/15409cdf862c4c1c818630021b594bc4, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/5a825039295e475f9c490d963431fe95, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/57153e3385324478a445b86c7bd76a41] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp, totalSize=100.5 K 2024-12-06T16:02:28,658 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:28,658 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. files: [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/15409cdf862c4c1c818630021b594bc4, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/5a825039295e475f9c490d963431fe95, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/57153e3385324478a445b86c7bd76a41] 2024-12-06T16:02:28,659 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2b4b4f4bcd245648a9a3ad0e88f10e3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733500943904 2024-12-06T16:02:28,659 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 15409cdf862c4c1c818630021b594bc4, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733500943904 2024-12-06T16:02:28,659 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a825039295e475f9c490d963431fe95, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733500945108 2024-12-06T16:02:28,659 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting db623fb025cd4576a7a4bbb9f082b9c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733500945108 2024-12-06T16:02:28,659 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 57153e3385324478a445b86c7bd76a41, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733500946313 2024-12-06T16:02:28,659 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 781d7663cd574043a6e58a803c70b123, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733500946313 2024-12-06T16:02:28,674 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81761a3d64f4403adedb17ba781753d2#C#compaction#384 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:28,675 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/03ed9adb11e145cbaee25b52ac5722be is 50, key is test_row_0/C:col10/1733500946976/Put/seqid=0 2024-12-06T16:02:28,676 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:28,685 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206633f5655d2454b00b9d44b29f84dd701_81761a3d64f4403adedb17ba781753d2 store=[table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:28,687 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206633f5655d2454b00b9d44b29f84dd701_81761a3d64f4403adedb17ba781753d2, store=[table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:28,688 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206633f5655d2454b00b9d44b29f84dd701_81761a3d64f4403adedb17ba781753d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:28,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742281_1457 (size=13017) 2024-12-06T16:02:28,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742282_1458 (size=4469) 2024-12-06T16:02:28,699 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81761a3d64f4403adedb17ba781753d2#A#compaction#385 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:28,699 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/de307e8f7bde430c8e37730489e8aedf is 175, key is test_row_0/A:col10/1733500946976/Put/seqid=0 2024-12-06T16:02:28,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742283_1459 (size=31971) 2024-12-06T16:02:28,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-06T16:02:28,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:28,805 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 81761a3d64f4403adedb17ba781753d2 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-06T16:02:28,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120656158857d5ba4d7d96a501a029035928_81761a3d64f4403adedb17ba781753d2 is 50, key is test_row_0/A:col10/1733500947012/Put/seqid=0 2024-12-06T16:02:28,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742284_1460 (size=12454) 2024-12-06T16:02:29,012 INFO [Thread-1691 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1691. 2024-12-06T16:02:29,012 DEBUG [Thread-1691 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:02:29,012 DEBUG [Thread-1691 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:29,012 DEBUG [Thread-1691 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:29,012 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:02:29,016 INFO [Thread-1689 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1689. 2024-12-06T16:02:29,016 INFO [Thread-1693 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1693. 2024-12-06T16:02:29,017 DEBUG [Thread-1689 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:02:29,017 DEBUG [Thread-1693 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:02:29,017 DEBUG [Thread-1689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:29,017 DEBUG [Thread-1693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:29,017 DEBUG [Thread-1693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:29,017 DEBUG [Thread-1689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:29,017 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:02:29,017 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:02:29,020 INFO [Thread-1695 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1695. 2024-12-06T16:02:29,020 DEBUG [Thread-1695 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:02:29,021 DEBUG [Thread-1695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:29,021 DEBUG [Thread-1695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:29,021 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:02:29,024 INFO [Thread-1697 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1697. 2024-12-06T16:02:29,024 DEBUG [Thread-1697 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:02:29,024 DEBUG [Thread-1697 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:29,025 DEBUG [Thread-1697 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:29,025 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:02:29,094 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/03ed9adb11e145cbaee25b52ac5722be as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/03ed9adb11e145cbaee25b52ac5722be 2024-12-06T16:02:29,098 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81761a3d64f4403adedb17ba781753d2/C of 81761a3d64f4403adedb17ba781753d2 into 03ed9adb11e145cbaee25b52ac5722be(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:29,098 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:29,099 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., storeName=81761a3d64f4403adedb17ba781753d2/C, priority=13, startTime=1733500948656; duration=0sec 2024-12-06T16:02:29,099 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:29,099 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81761a3d64f4403adedb17ba781753d2:C 2024-12-06T16:02:29,099 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:29,099 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:29,100 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): 81761a3d64f4403adedb17ba781753d2/B is initiating minor compaction (all files) 2024-12-06T16:02:29,100 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81761a3d64f4403adedb17ba781753d2/B in TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:29,100 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/8b624e0654694974987ca8a9b03a8cf1, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/cd8e12fc146e450a94ea1f530a154287, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/622016923b17458196436099dc1d1fa3] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp, totalSize=36.5 K 2024-12-06T16:02:29,100 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b624e0654694974987ca8a9b03a8cf1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733500943904 2024-12-06T16:02:29,100 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd8e12fc146e450a94ea1f530a154287, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733500945108 2024-12-06T16:02:29,101 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 622016923b17458196436099dc1d1fa3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733500946313 2024-12-06T16:02:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-06T16:02:29,106 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81761a3d64f4403adedb17ba781753d2#B#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:29,106 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/29b00bc4a67e46d1a49dd9f39c16f842 is 50, key is test_row_0/B:col10/1733500946976/Put/seqid=0 2024-12-06T16:02:29,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742285_1461 (size=13017) 2024-12-06T16:02:29,112 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/de307e8f7bde430c8e37730489e8aedf as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/de307e8f7bde430c8e37730489e8aedf 2024-12-06T16:02:29,116 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81761a3d64f4403adedb17ba781753d2/A of 81761a3d64f4403adedb17ba781753d2 into de307e8f7bde430c8e37730489e8aedf(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:29,116 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:29,116 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., storeName=81761a3d64f4403adedb17ba781753d2/A, priority=13, startTime=1733500948656; duration=0sec 2024-12-06T16:02:29,116 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:29,116 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81761a3d64f4403adedb17ba781753d2:A 2024-12-06T16:02:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:29,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. as already flushing 2024-12-06T16:02:29,205 INFO [Thread-1686 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1686. 2024-12-06T16:02:29,205 DEBUG [Thread-1686 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:02:29,205 INFO [Thread-1680 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1680. 2024-12-06T16:02:29,205 DEBUG [Thread-1686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:29,205 DEBUG [Thread-1680 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:02:29,205 DEBUG [Thread-1680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:29,205 DEBUG [Thread-1686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:29,205 DEBUG [Thread-1680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:29,205 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:02:29,205 INFO [Thread-1678 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1678. 2024-12-06T16:02:29,206 DEBUG [Thread-1678 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:02:29,206 DEBUG [Thread-1678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:29,206 DEBUG [Thread-1678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:29,206 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:02:29,206 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:02:29,207 INFO [Thread-1684 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1684. 2024-12-06T16:02:29,207 DEBUG [Thread-1684 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:02:29,207 DEBUG [Thread-1684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:29,207 DEBUG [Thread-1684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:29,207 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:02:29,207 INFO [Thread-1682 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1682. 2024-12-06T16:02:29,207 DEBUG [Thread-1682 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:02:29,207 DEBUG [Thread-1682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:29,207 DEBUG [Thread-1682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:29,207 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:02:29,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:02:29,230 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120656158857d5ba4d7d96a501a029035928_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120656158857d5ba4d7d96a501a029035928_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:29,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/4fc630a2d0054da78dcb205d1dfa1feb, store: [table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:29,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/4fc630a2d0054da78dcb205d1dfa1feb is 175, key is test_row_0/A:col10/1733500947012/Put/seqid=0 2024-12-06T16:02:29,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742286_1462 (size=31255) 2024-12-06T16:02:29,515 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/29b00bc4a67e46d1a49dd9f39c16f842 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/29b00bc4a67e46d1a49dd9f39c16f842 2024-12-06T16:02:29,519 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81761a3d64f4403adedb17ba781753d2/B of 81761a3d64f4403adedb17ba781753d2 into 29b00bc4a67e46d1a49dd9f39c16f842(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:29,519 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:29,519 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., storeName=81761a3d64f4403adedb17ba781753d2/B, priority=13, startTime=1733500948656; duration=0sec 2024-12-06T16:02:29,519 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:29,519 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81761a3d64f4403adedb17ba781753d2:B 2024-12-06T16:02:29,636 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/4fc630a2d0054da78dcb205d1dfa1feb 2024-12-06T16:02:29,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/433aa1221d884ec7b9bfd8126dad672f is 50, key is test_row_0/B:col10/1733500947012/Put/seqid=0 2024-12-06T16:02:29,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742287_1463 (size=12301) 2024-12-06T16:02:30,046 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/433aa1221d884ec7b9bfd8126dad672f 2024-12-06T16:02:30,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/9bfd39c5c3a945d7a0f3ce258dfb4559 is 50, key is test_row_0/C:col10/1733500947012/Put/seqid=0 2024-12-06T16:02:30,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742288_1464 (size=12301) 2024-12-06T16:02:30,456 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/9bfd39c5c3a945d7a0f3ce258dfb4559 2024-12-06T16:02:30,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/4fc630a2d0054da78dcb205d1dfa1feb as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/4fc630a2d0054da78dcb205d1dfa1feb 2024-12-06T16:02:30,464 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/4fc630a2d0054da78dcb205d1dfa1feb, entries=150, sequenceid=312, filesize=30.5 K 2024-12-06T16:02:30,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/433aa1221d884ec7b9bfd8126dad672f as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/433aa1221d884ec7b9bfd8126dad672f 2024-12-06T16:02:30,468 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/433aa1221d884ec7b9bfd8126dad672f, entries=150, sequenceid=312, filesize=12.0 K 2024-12-06T16:02:30,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/9bfd39c5c3a945d7a0f3ce258dfb4559 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/9bfd39c5c3a945d7a0f3ce258dfb4559 2024-12-06T16:02:30,473 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/9bfd39c5c3a945d7a0f3ce258dfb4559, entries=150, sequenceid=312, filesize=12.0 K 2024-12-06T16:02:30,474 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=33.54 KB/34350 for 81761a3d64f4403adedb17ba781753d2 in 1669ms, sequenceid=312, compaction requested=false 2024-12-06T16:02:30,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 81761a3d64f4403adedb17ba781753d2: 2024-12-06T16:02:30,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:30,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-06T16:02:30,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-06T16:02:30,477 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-06T16:02:30,477 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5190 sec 2024-12-06T16:02:30,479 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 3.5250 sec 2024-12-06T16:02:30,507 WARN [regionserver/85bef17d9292:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-12-06T16:02:30,509 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/e44d3f2ecfb2477da322fe47fdaa47c1, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/973c88d96816417f9505b3d7b2d4274c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/1fbdd0db3c8544be8527c5477e70bf36, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/e96ee7e94acf4dd99ca423a16be2b9a2, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/17f7452fa5ba48579b8c097a7ae0a162, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/b81ecf0d5db64dce962b502185fc5d45, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/3d517f61816f4fbb9c74ffef162c4e6d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/ed772275f8464d0e82ede86b31bdf162, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/c95f3c41067840368d9280f00e4783a3, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/6d4b42ca541f4fc28e68a5d859365a5a, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/d2860461e3b74111ad08dfb10419da05, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/cc484b3cfc6b4d4a9e0ddc3afef04544, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/7eda3659f24f409291b58710d11c3e27, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/38cb136457634cefb2f4eb2f36cadfa5, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/06c9fbb162d44ca6a188a79b07aacfd6, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/a16d83b3705a4667a8e6516527417926, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/db95458be79d45f6808eac29ed79fa14, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/5a8a531e0a5c4112b4d7913ae1063af8, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/15409cdf862c4c1c818630021b594bc4, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/5a825039295e475f9c490d963431fe95, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/57153e3385324478a445b86c7bd76a41] to archive 2024-12-06T16:02:30,510 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T16:02:30,512 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/e44d3f2ecfb2477da322fe47fdaa47c1 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/e44d3f2ecfb2477da322fe47fdaa47c1 2024-12-06T16:02:30,513 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/973c88d96816417f9505b3d7b2d4274c to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/973c88d96816417f9505b3d7b2d4274c 2024-12-06T16:02:30,514 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/1fbdd0db3c8544be8527c5477e70bf36 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/1fbdd0db3c8544be8527c5477e70bf36 2024-12-06T16:02:30,515 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/e96ee7e94acf4dd99ca423a16be2b9a2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/e96ee7e94acf4dd99ca423a16be2b9a2 2024-12-06T16:02:30,516 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/17f7452fa5ba48579b8c097a7ae0a162 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/17f7452fa5ba48579b8c097a7ae0a162 2024-12-06T16:02:30,517 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/b81ecf0d5db64dce962b502185fc5d45 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/b81ecf0d5db64dce962b502185fc5d45 2024-12-06T16:02:30,518 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/3d517f61816f4fbb9c74ffef162c4e6d to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/3d517f61816f4fbb9c74ffef162c4e6d 2024-12-06T16:02:30,519 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/ed772275f8464d0e82ede86b31bdf162 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/ed772275f8464d0e82ede86b31bdf162 2024-12-06T16:02:30,520 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/c95f3c41067840368d9280f00e4783a3 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/c95f3c41067840368d9280f00e4783a3 2024-12-06T16:02:30,521 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/6d4b42ca541f4fc28e68a5d859365a5a to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/6d4b42ca541f4fc28e68a5d859365a5a 2024-12-06T16:02:30,522 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/d2860461e3b74111ad08dfb10419da05 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/d2860461e3b74111ad08dfb10419da05 2024-12-06T16:02:30,523 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/cc484b3cfc6b4d4a9e0ddc3afef04544 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/cc484b3cfc6b4d4a9e0ddc3afef04544 2024-12-06T16:02:30,523 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/7eda3659f24f409291b58710d11c3e27 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/7eda3659f24f409291b58710d11c3e27 2024-12-06T16:02:30,525 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/38cb136457634cefb2f4eb2f36cadfa5 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/38cb136457634cefb2f4eb2f36cadfa5 2024-12-06T16:02:30,526 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/06c9fbb162d44ca6a188a79b07aacfd6 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/06c9fbb162d44ca6a188a79b07aacfd6 2024-12-06T16:02:30,527 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/a16d83b3705a4667a8e6516527417926 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/a16d83b3705a4667a8e6516527417926 2024-12-06T16:02:30,528 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/db95458be79d45f6808eac29ed79fa14 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/db95458be79d45f6808eac29ed79fa14 2024-12-06T16:02:30,529 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/5a8a531e0a5c4112b4d7913ae1063af8 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/5a8a531e0a5c4112b4d7913ae1063af8 2024-12-06T16:02:30,530 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/15409cdf862c4c1c818630021b594bc4 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/15409cdf862c4c1c818630021b594bc4 2024-12-06T16:02:30,531 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/5a825039295e475f9c490d963431fe95 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/5a825039295e475f9c490d963431fe95 2024-12-06T16:02:30,533 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/57153e3385324478a445b86c7bd76a41 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/57153e3385324478a445b86c7bd76a41 2024-12-06T16:02:30,536 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/fece3394285f43778b8e59362d6471c5, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/a17672fd865c489d9c8bc39f5e85a4c4, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/722de527f72044b5b31d5b778289a401, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/b35322c397ba4b04b5e7ac3f402bf6bb, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/6db4ae9873e847699542de1de033c87d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/b80e3a0c72f84a7ba188be567764005a, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/65e57d05727642c094ffbc91085296e0, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/614d29bd6ac546598a7fdc5d8405cd5e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/9df178cc4f3f4963871b88fc2e71591e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/1f4f4245343d40baa654015fad5d4915, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/b98914dc6151451fb947e539de9266f0, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/170110204f6c448e8c7700d36cc868bd, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/0c55f3e579504521ab8416b76c4577c4, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/0171733e98fa400b9186f750e2868fd4, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/3af9e327dc024a38adc185d52999b960, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/548f4f78c1a1415087cef8823dddafa3, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/3f14c595f76a4a0ebdbfaeee5d4d44ba, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/8b624e0654694974987ca8a9b03a8cf1, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/489c176aceba4cd391684f482ef8708c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/cd8e12fc146e450a94ea1f530a154287, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/622016923b17458196436099dc1d1fa3] to archive 2024-12-06T16:02:30,537 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T16:02:30,538 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/fece3394285f43778b8e59362d6471c5 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/fece3394285f43778b8e59362d6471c5 2024-12-06T16:02:30,539 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/a17672fd865c489d9c8bc39f5e85a4c4 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/a17672fd865c489d9c8bc39f5e85a4c4 2024-12-06T16:02:30,540 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/722de527f72044b5b31d5b778289a401 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/722de527f72044b5b31d5b778289a401 2024-12-06T16:02:30,541 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/b35322c397ba4b04b5e7ac3f402bf6bb to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/b35322c397ba4b04b5e7ac3f402bf6bb 2024-12-06T16:02:30,542 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/6db4ae9873e847699542de1de033c87d to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/6db4ae9873e847699542de1de033c87d 2024-12-06T16:02:30,543 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/b80e3a0c72f84a7ba188be567764005a to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/b80e3a0c72f84a7ba188be567764005a 2024-12-06T16:02:30,544 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/65e57d05727642c094ffbc91085296e0 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/65e57d05727642c094ffbc91085296e0 2024-12-06T16:02:30,545 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/614d29bd6ac546598a7fdc5d8405cd5e to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/614d29bd6ac546598a7fdc5d8405cd5e 2024-12-06T16:02:30,546 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/9df178cc4f3f4963871b88fc2e71591e to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/9df178cc4f3f4963871b88fc2e71591e 2024-12-06T16:02:30,547 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/1f4f4245343d40baa654015fad5d4915 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/1f4f4245343d40baa654015fad5d4915 2024-12-06T16:02:30,548 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/b98914dc6151451fb947e539de9266f0 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/b98914dc6151451fb947e539de9266f0 2024-12-06T16:02:30,549 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/170110204f6c448e8c7700d36cc868bd to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/170110204f6c448e8c7700d36cc868bd 2024-12-06T16:02:30,550 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/0c55f3e579504521ab8416b76c4577c4 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/0c55f3e579504521ab8416b76c4577c4 2024-12-06T16:02:30,551 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/0171733e98fa400b9186f750e2868fd4 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/0171733e98fa400b9186f750e2868fd4 2024-12-06T16:02:30,552 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/3af9e327dc024a38adc185d52999b960 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/3af9e327dc024a38adc185d52999b960 2024-12-06T16:02:30,553 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/548f4f78c1a1415087cef8823dddafa3 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/548f4f78c1a1415087cef8823dddafa3 2024-12-06T16:02:30,554 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/3f14c595f76a4a0ebdbfaeee5d4d44ba to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/3f14c595f76a4a0ebdbfaeee5d4d44ba 2024-12-06T16:02:30,555 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/8b624e0654694974987ca8a9b03a8cf1 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/8b624e0654694974987ca8a9b03a8cf1 2024-12-06T16:02:30,556 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/489c176aceba4cd391684f482ef8708c to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/489c176aceba4cd391684f482ef8708c 2024-12-06T16:02:30,557 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/cd8e12fc146e450a94ea1f530a154287 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/cd8e12fc146e450a94ea1f530a154287 2024-12-06T16:02:30,558 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/622016923b17458196436099dc1d1fa3 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/622016923b17458196436099dc1d1fa3 2024-12-06T16:02:30,562 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/20f4928bb1384e508acd271c95561857, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/00a9f63103c64de397a453390ea8c3a2, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/82ac452b72604396afaf2e0423cf81b7, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/bc89ac95337f401ab1da32d00f95d78a, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/5d1f9c3b06b0481bab645af64e3b54be, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/ea1f00667e1f4997ba2f8b59579bd4e2, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/6f2f509c81014f02ad70d84f143c7d82, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/b6ca1d1adc9948efbeb24c4ba6127959, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/5726587425d74704a7829dd5dec67235, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/fa940d57dea64247b64848ae358e02ac, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/299b042d850949dc873ea3e0a96439c8, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/ce90db3619cc4529ba7f4faa1588eb84, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/5b3dd7dc9e8244a28c946dc53404d2c3, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/da1807f35a8647238c4f7e27254f5a7d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/f1575e3b55004dc5a5b6662993c41142, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/63eb4bb05c5a40778eb8fb2744db461e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/ea3b71a81dd94fcd8f5539e8b079efda, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/c2b4b4f4bcd245648a9a3ad0e88f10e3, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/5ef7e58fc1f3412e9a8c8541b126f815, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/db623fb025cd4576a7a4bbb9f082b9c8, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/781d7663cd574043a6e58a803c70b123] to archive 2024-12-06T16:02:30,562 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T16:02:30,564 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/20f4928bb1384e508acd271c95561857 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/20f4928bb1384e508acd271c95561857 2024-12-06T16:02:30,565 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/00a9f63103c64de397a453390ea8c3a2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/00a9f63103c64de397a453390ea8c3a2 2024-12-06T16:02:30,565 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/82ac452b72604396afaf2e0423cf81b7 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/82ac452b72604396afaf2e0423cf81b7 2024-12-06T16:02:30,566 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/bc89ac95337f401ab1da32d00f95d78a to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/bc89ac95337f401ab1da32d00f95d78a 2024-12-06T16:02:30,567 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/5d1f9c3b06b0481bab645af64e3b54be to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/5d1f9c3b06b0481bab645af64e3b54be 2024-12-06T16:02:30,568 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/ea1f00667e1f4997ba2f8b59579bd4e2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/ea1f00667e1f4997ba2f8b59579bd4e2 2024-12-06T16:02:30,568 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/6f2f509c81014f02ad70d84f143c7d82 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/6f2f509c81014f02ad70d84f143c7d82 2024-12-06T16:02:30,569 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/b6ca1d1adc9948efbeb24c4ba6127959 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/b6ca1d1adc9948efbeb24c4ba6127959 2024-12-06T16:02:30,570 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/5726587425d74704a7829dd5dec67235 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/5726587425d74704a7829dd5dec67235 2024-12-06T16:02:30,571 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/fa940d57dea64247b64848ae358e02ac to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/fa940d57dea64247b64848ae358e02ac 2024-12-06T16:02:30,571 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/299b042d850949dc873ea3e0a96439c8 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/299b042d850949dc873ea3e0a96439c8 2024-12-06T16:02:30,572 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/ce90db3619cc4529ba7f4faa1588eb84 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/ce90db3619cc4529ba7f4faa1588eb84 2024-12-06T16:02:30,573 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/5b3dd7dc9e8244a28c946dc53404d2c3 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/5b3dd7dc9e8244a28c946dc53404d2c3 2024-12-06T16:02:30,574 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/da1807f35a8647238c4f7e27254f5a7d to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/da1807f35a8647238c4f7e27254f5a7d 2024-12-06T16:02:30,574 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/f1575e3b55004dc5a5b6662993c41142 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/f1575e3b55004dc5a5b6662993c41142 2024-12-06T16:02:30,575 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/63eb4bb05c5a40778eb8fb2744db461e to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/63eb4bb05c5a40778eb8fb2744db461e 2024-12-06T16:02:30,576 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/ea3b71a81dd94fcd8f5539e8b079efda to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/ea3b71a81dd94fcd8f5539e8b079efda 2024-12-06T16:02:30,577 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/c2b4b4f4bcd245648a9a3ad0e88f10e3 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/c2b4b4f4bcd245648a9a3ad0e88f10e3 2024-12-06T16:02:30,578 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/5ef7e58fc1f3412e9a8c8541b126f815 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/5ef7e58fc1f3412e9a8c8541b126f815 2024-12-06T16:02:30,578 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/db623fb025cd4576a7a4bbb9f082b9c8 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/db623fb025cd4576a7a4bbb9f082b9c8 2024-12-06T16:02:30,579 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/85bef17d9292:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/781d7663cd574043a6e58a803c70b123 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/781d7663cd574043a6e58a803c70b123 2024-12-06T16:02:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-06T16:02:31,122 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:02:31,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-06T16:02:31,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-12-06T16:02:31,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-12-06T16:02:31,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-12-06T16:02:31,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-12-06T16:02:31,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-12-06T16:02:31,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-06T16:02:31,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-06T16:02:31,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1055 2024-12-06T16:02:31,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3164 rows 2024-12-06T16:02:31,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1053 2024-12-06T16:02:31,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3158 rows 2024-12-06T16:02:31,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1054 2024-12-06T16:02:31,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3158 rows 2024-12-06T16:02:31,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1057 2024-12-06T16:02:31,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3170 rows 2024-12-06T16:02:31,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1059 2024-12-06T16:02:31,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3176 rows 2024-12-06T16:02:31,123 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T16:02:31,123 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T16:02:31,123 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:31,123 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:31,123 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:02:31,124 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-06T16:02:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-06T16:02:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-06T16:02:31,127 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733500951126"}]},"ts":"1733500951126"} 2024-12-06T16:02:31,128 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-06T16:02:31,128 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-06T16:02:31,129 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-06T16:02:31,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81761a3d64f4403adedb17ba781753d2, UNASSIGN}] 2024-12-06T16:02:31,131 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=101, ppid=100, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81761a3d64f4403adedb17ba781753d2, UNASSIGN 2024-12-06T16:02:31,131 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=101 updating hbase:meta row=81761a3d64f4403adedb17ba781753d2, regionState=CLOSING, regionLocation=85bef17d9292,46105,1733500827732 2024-12-06T16:02:31,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=101, ppid=100, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=81761a3d64f4403adedb17ba781753d2, UNASSIGN because future has completed 2024-12-06T16:02:31,133 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-06T16:02:31,133 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; CloseRegionProcedure 81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732}] 2024-12-06T16:02:31,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-06T16:02:31,285 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] handler.UnassignRegionHandler(122): Close 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:31,285 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-06T16:02:31,285 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1722): Closing 81761a3d64f4403adedb17ba781753d2, disabling compactions & flushes 2024-12-06T16:02:31,285 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:31,285 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:31,285 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. after waiting 0 ms 2024-12-06T16:02:31,285 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:31,285 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(2902): Flushing 81761a3d64f4403adedb17ba781753d2 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-12-06T16:02:31,290 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412061fb7ff54a0194ee0b63ee97a47eddf55_81761a3d64f4403adedb17ba781753d2 is 50, key is test_row_0/A:col10/1733500949204/Put/seqid=0 2024-12-06T16:02:31,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742289_1465 (size=12454) 2024-12-06T16:02:31,295 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:02:31,298 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412061fb7ff54a0194ee0b63ee97a47eddf55_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412061fb7ff54a0194ee0b63ee97a47eddf55_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:31,299 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/b4f54f75350341718986859c39b9b61e, store: [table=TestAcidGuarantees family=A region=81761a3d64f4403adedb17ba781753d2] 2024-12-06T16:02:31,299 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/b4f54f75350341718986859c39b9b61e is 175, key is test_row_0/A:col10/1733500949204/Put/seqid=0 2024-12-06T16:02:31,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742290_1466 (size=31255) 2024-12-06T16:02:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-06T16:02:31,703 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/b4f54f75350341718986859c39b9b61e 2024-12-06T16:02:31,709 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/1f9a44e5f19c4ac1805ab23ca41f3464 is 50, key is test_row_0/B:col10/1733500949204/Put/seqid=0 2024-12-06T16:02:31,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742291_1467 (size=12301) 2024-12-06T16:02:31,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-06T16:02:32,112 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/1f9a44e5f19c4ac1805ab23ca41f3464 2024-12-06T16:02:32,118 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/c621c3ee3bc447cb912240f295493191 is 50, key is test_row_0/C:col10/1733500949204/Put/seqid=0 2024-12-06T16:02:32,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742292_1468 (size=12301) 2024-12-06T16:02:32,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-06T16:02:32,522 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/c621c3ee3bc447cb912240f295493191 2024-12-06T16:02:32,526 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/A/b4f54f75350341718986859c39b9b61e as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/b4f54f75350341718986859c39b9b61e 2024-12-06T16:02:32,529 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/b4f54f75350341718986859c39b9b61e, entries=150, sequenceid=323, filesize=30.5 K 2024-12-06T16:02:32,530 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/B/1f9a44e5f19c4ac1805ab23ca41f3464 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/1f9a44e5f19c4ac1805ab23ca41f3464 2024-12-06T16:02:32,533 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/1f9a44e5f19c4ac1805ab23ca41f3464, entries=150, sequenceid=323, filesize=12.0 K 2024-12-06T16:02:32,534 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/.tmp/C/c621c3ee3bc447cb912240f295493191 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/c621c3ee3bc447cb912240f295493191 2024-12-06T16:02:32,537 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/c621c3ee3bc447cb912240f295493191, entries=150, sequenceid=323, filesize=12.0 K 2024-12-06T16:02:32,538 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 81761a3d64f4403adedb17ba781753d2 in 1253ms, sequenceid=323, compaction requested=true 2024-12-06T16:02:32,542 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/recovered.edits/326.seqid, newMaxSeqId=326, maxSeqId=4 2024-12-06T16:02:32,543 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. 2024-12-06T16:02:32,543 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1676): Region close journal for 81761a3d64f4403adedb17ba781753d2: Waiting for close lock at 1733500951285Running coprocessor pre-close hooks at 1733500951285Disabling compacts and flushes for region at 1733500951285Disabling writes for close at 1733500951285Obtaining lock to block concurrent updates at 1733500951285Preparing flush snapshotting stores in 81761a3d64f4403adedb17ba781753d2 at 1733500951285Finished memstore snapshotting TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1733500951286 (+1 ms)Flushing stores of TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2. at 1733500951286Flushing 81761a3d64f4403adedb17ba781753d2/A: creating writer at 1733500951286Flushing mob file 81761a3d64f4403adedb17ba781753d2/A: appending metadata at 1733500951290 (+4 ms)Flushing mob file 81761a3d64f4403adedb17ba781753d2/A: closing flushed file at 1733500951290Flushing 81761a3d64f4403adedb17ba781753d2/A: appending metadata at 1733500951299 (+9 ms)Flushing 81761a3d64f4403adedb17ba781753d2/A: closing flushed file at 1733500951299Flushing 81761a3d64f4403adedb17ba781753d2/B: creating writer at 1733500951706 (+407 ms)Flushing 81761a3d64f4403adedb17ba781753d2/B: appending metadata at 1733500951708 (+2 ms)Flushing 81761a3d64f4403adedb17ba781753d2/B: closing flushed file at 1733500951708Flushing 81761a3d64f4403adedb17ba781753d2/C: creating writer at 1733500952116 (+408 ms)Flushing 81761a3d64f4403adedb17ba781753d2/C: appending metadata at 1733500952118 (+2 ms)Flushing 81761a3d64f4403adedb17ba781753d2/C: closing flushed file at 1733500952118Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@426a3799: reopening flushed file at 1733500952525 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b36c74e: reopening flushed file at 1733500952529 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a3f1c14: reopening flushed file at 1733500952533 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 81761a3d64f4403adedb17ba781753d2 in 1253ms, sequenceid=323, compaction requested=true at 1733500952538 (+5 ms)Writing region close event to WAL at 1733500952540 (+2 ms)Running coprocessor post-close hooks at 1733500952543 (+3 ms)Closed at 1733500952543 2024-12-06T16:02:32,545 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] handler.UnassignRegionHandler(157): Closed 81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:32,546 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=101 updating hbase:meta row=81761a3d64f4403adedb17ba781753d2, regionState=CLOSED 2024-12-06T16:02:32,547 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE, hasLock=false; CloseRegionProcedure 81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 because future has completed 2024-12-06T16:02:32,549 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-06T16:02:32,549 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; CloseRegionProcedure 81761a3d64f4403adedb17ba781753d2, server=85bef17d9292,46105,1733500827732 in 1.4150 sec 2024-12-06T16:02:32,551 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-12-06T16:02:32,551 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81761a3d64f4403adedb17ba781753d2, UNASSIGN in 1.4190 sec 2024-12-06T16:02:32,553 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-06T16:02:32,553 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4230 sec 2024-12-06T16:02:32,554 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733500952554"}]},"ts":"1733500952554"} 2024-12-06T16:02:32,555 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-06T16:02:32,555 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-06T16:02:32,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4320 sec 2024-12-06T16:02:33,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-06T16:02:33,272 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:02:33,273 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-06T16:02:33,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:02:33,275 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=103, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:02:33,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-06T16:02:33,275 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=103, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:02:33,277 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:33,279 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A, FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B, FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C, FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/recovered.edits] 2024-12-06T16:02:33,281 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/4fc630a2d0054da78dcb205d1dfa1feb to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/4fc630a2d0054da78dcb205d1dfa1feb 2024-12-06T16:02:33,282 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/b4f54f75350341718986859c39b9b61e to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/b4f54f75350341718986859c39b9b61e 2024-12-06T16:02:33,283 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/de307e8f7bde430c8e37730489e8aedf to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/A/de307e8f7bde430c8e37730489e8aedf 2024-12-06T16:02:33,286 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/1f9a44e5f19c4ac1805ab23ca41f3464 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/1f9a44e5f19c4ac1805ab23ca41f3464 2024-12-06T16:02:33,287 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/29b00bc4a67e46d1a49dd9f39c16f842 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/29b00bc4a67e46d1a49dd9f39c16f842 2024-12-06T16:02:33,288 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/433aa1221d884ec7b9bfd8126dad672f to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/B/433aa1221d884ec7b9bfd8126dad672f 2024-12-06T16:02:33,290 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/03ed9adb11e145cbaee25b52ac5722be to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/03ed9adb11e145cbaee25b52ac5722be 2024-12-06T16:02:33,291 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/9bfd39c5c3a945d7a0f3ce258dfb4559 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/9bfd39c5c3a945d7a0f3ce258dfb4559 2024-12-06T16:02:33,292 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/c621c3ee3bc447cb912240f295493191 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/C/c621c3ee3bc447cb912240f295493191 2024-12-06T16:02:33,294 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/recovered.edits/326.seqid to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2/recovered.edits/326.seqid 2024-12-06T16:02:33,294 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:33,295 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-06T16:02:33,295 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-06T16:02:33,296 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-06T16:02:33,298 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412061fb7ff54a0194ee0b63ee97a47eddf55_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412061fb7ff54a0194ee0b63ee97a47eddf55_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:33,299 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206289ba9b0398a4dd7856fc4d47b40f714_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206289ba9b0398a4dd7856fc4d47b40f714_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:33,300 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412062c96ef84c58e4cc39ddd6ddd14f633d4_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412062c96ef84c58e4cc39ddd6ddd14f633d4_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:33,301 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412062d932b68e39b42fdbd3ff6074a149a0e_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412062d932b68e39b42fdbd3ff6074a149a0e_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:33,302 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412062f0c898f98074a138803554b77e252d1_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412062f0c898f98074a138803554b77e252d1_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:33,303 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120656158857d5ba4d7d96a501a029035928_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120656158857d5ba4d7d96a501a029035928_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:33,304 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120658ee84c1fcbe439fac0998c5b4af07a8_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120658ee84c1fcbe439fac0998c5b4af07a8_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:33,305 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206641ff027602e4a25b09120af9e3bf9ee_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206641ff027602e4a25b09120af9e3bf9ee_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:33,306 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412066e053c0c7b3347ec995d1aa4ce81c3c2_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412066e053c0c7b3347ec995d1aa4ce81c3c2_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:33,306 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412067305347d67564b8b91b3954446e63608_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412067305347d67564b8b91b3954446e63608_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:33,307 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120678de496b507f499990e5013099106f82_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120678de496b507f499990e5013099106f82_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:33,308 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206822156c6e814433a9cad5aca4faba897_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206822156c6e814433a9cad5aca4faba897_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:33,309 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a16ba1e2087b410fb6c40191badf2bf4_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a16ba1e2087b410fb6c40191badf2bf4_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:33,310 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a1b1b213145a4cfd8000a5aa338cc3ca_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a1b1b213145a4cfd8000a5aa338cc3ca_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:33,311 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206bf7b190fb3be4bdbb7ec64e383b12b05_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206bf7b190fb3be4bdbb7ec64e383b12b05_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:33,312 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206c4f21a94291142829502f610990b27f5_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206c4f21a94291142829502f610990b27f5_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:33,313 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206f66c83e1557146d180ff7bca7feecebe_81761a3d64f4403adedb17ba781753d2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206f66c83e1557146d180ff7bca7feecebe_81761a3d64f4403adedb17ba781753d2 2024-12-06T16:02:33,314 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-06T16:02:33,315 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=103, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:02:33,317 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-06T16:02:33,319 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-06T16:02:33,320 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=103, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:02:33,320 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-06T16:02:33,320 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733500953320"}]},"ts":"9223372036854775807"} 2024-12-06T16:02:33,322 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-06T16:02:33,322 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 81761a3d64f4403adedb17ba781753d2, NAME => 'TestAcidGuarantees,,1733500925891.81761a3d64f4403adedb17ba781753d2.', STARTKEY => '', ENDKEY => ''}] 2024-12-06T16:02:33,322 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-06T16:02:33,322 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733500953322"}]},"ts":"9223372036854775807"} 2024-12-06T16:02:33,324 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-06T16:02:33,325 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=103, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:02:33,325 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 52 msec 2024-12-06T16:02:33,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-06T16:02:33,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-06T16:02:33,382 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:02:33,392 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240 (was 238) - Thread LEAK? -, OpenFileDescriptor=451 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=490 (was 519), ProcessCount=11 (was 11), AvailableMemoryMB=8656 (was 8694) 2024-12-06T16:02:33,400 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=490, ProcessCount=11, AvailableMemoryMB=8656 2024-12-06T16:02:33,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-06T16:02:33,402 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T16:02:33,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=104, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-06T16:02:33,404 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T16:02:33,405 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:02:33,405 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 104 2024-12-06T16:02:33,405 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T16:02:33,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-12-06T16:02:33,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742293_1469 (size=959) 2024-12-06T16:02:33,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-12-06T16:02:33,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-12-06T16:02:33,812 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d 2024-12-06T16:02:33,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742294_1470 (size=53) 2024-12-06T16:02:33,817 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T16:02:33,817 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d252d31b6d43033dcd4545dc84403a2e, disabling compactions & flushes 2024-12-06T16:02:33,817 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:33,817 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:33,817 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. after waiting 0 ms 2024-12-06T16:02:33,817 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:33,817 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:33,817 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d252d31b6d43033dcd4545dc84403a2e: Waiting for close lock at 1733500953817Disabling compacts and flushes for region at 1733500953817Disabling writes for close at 1733500953817Writing region close event to WAL at 1733500953817Closed at 1733500953817 2024-12-06T16:02:33,818 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T16:02:33,818 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733500953818"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733500953818"}]},"ts":"1733500953818"} 2024-12-06T16:02:33,820 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T16:02:33,820 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T16:02:33,821 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733500953820"}]},"ts":"1733500953820"} 2024-12-06T16:02:33,822 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-06T16:02:33,822 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d252d31b6d43033dcd4545dc84403a2e, ASSIGN}] 2024-12-06T16:02:33,823 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d252d31b6d43033dcd4545dc84403a2e, ASSIGN 2024-12-06T16:02:33,823 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d252d31b6d43033dcd4545dc84403a2e, ASSIGN; state=OFFLINE, location=85bef17d9292,46105,1733500827732; forceNewPlan=false, retain=false 2024-12-06T16:02:33,974 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=d252d31b6d43033dcd4545dc84403a2e, regionState=OPENING, regionLocation=85bef17d9292,46105,1733500827732 2024-12-06T16:02:33,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d252d31b6d43033dcd4545dc84403a2e, ASSIGN because future has completed 2024-12-06T16:02:33,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; OpenRegionProcedure d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732}] 2024-12-06T16:02:34,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-12-06T16:02:34,130 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:34,131 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7752): Opening region: {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} 2024-12-06T16:02:34,132 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:34,132 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T16:02:34,132 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7794): checking encryption for d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:34,132 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7797): checking classloading for d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:34,133 INFO [StoreOpener-d252d31b6d43033dcd4545dc84403a2e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:34,134 INFO [StoreOpener-d252d31b6d43033dcd4545dc84403a2e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d252d31b6d43033dcd4545dc84403a2e columnFamilyName A 2024-12-06T16:02:34,134 DEBUG [StoreOpener-d252d31b6d43033dcd4545dc84403a2e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:02:34,134 INFO [StoreOpener-d252d31b6d43033dcd4545dc84403a2e-1 {}] regionserver.HStore(327): Store=d252d31b6d43033dcd4545dc84403a2e/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T16:02:34,135 INFO [StoreOpener-d252d31b6d43033dcd4545dc84403a2e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:34,135 INFO [StoreOpener-d252d31b6d43033dcd4545dc84403a2e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d252d31b6d43033dcd4545dc84403a2e columnFamilyName B 2024-12-06T16:02:34,135 DEBUG [StoreOpener-d252d31b6d43033dcd4545dc84403a2e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:02:34,136 INFO [StoreOpener-d252d31b6d43033dcd4545dc84403a2e-1 {}] regionserver.HStore(327): Store=d252d31b6d43033dcd4545dc84403a2e/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T16:02:34,136 INFO [StoreOpener-d252d31b6d43033dcd4545dc84403a2e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:34,137 INFO [StoreOpener-d252d31b6d43033dcd4545dc84403a2e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d252d31b6d43033dcd4545dc84403a2e columnFamilyName C 2024-12-06T16:02:34,137 DEBUG [StoreOpener-d252d31b6d43033dcd4545dc84403a2e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:02:34,137 INFO [StoreOpener-d252d31b6d43033dcd4545dc84403a2e-1 {}] regionserver.HStore(327): Store=d252d31b6d43033dcd4545dc84403a2e/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T16:02:34,137 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1038): replaying wal for d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:34,138 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:34,138 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:34,139 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1048): stopping wal replay for d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:34,139 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1060): Cleaning up temporary data for d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:34,139 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T16:02:34,140 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1093): writing seq id for d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:34,141 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T16:02:34,142 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1114): Opened d252d31b6d43033dcd4545dc84403a2e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72387674, jitterRate=0.07866039872169495}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T16:02:34,142 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:34,142 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1006): Region open journal for d252d31b6d43033dcd4545dc84403a2e: Running coprocessor pre-open hook at 1733500954132Writing region info on filesystem at 1733500954132Initializing all the Stores at 1733500954132Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733500954132Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733500954133 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733500954133Cleaning up temporary data from old regions at 1733500954139 (+6 ms)Running coprocessor post-open hooks at 1733500954142 (+3 ms)Region opened successfully at 1733500954142 2024-12-06T16:02:34,143 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., pid=106, masterSystemTime=1733500954128 2024-12-06T16:02:34,144 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:34,145 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:34,145 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=d252d31b6d43033dcd4545dc84403a2e, regionState=OPEN, openSeqNum=2, regionLocation=85bef17d9292,46105,1733500827732 2024-12-06T16:02:34,146 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=106, ppid=105, state=RUNNABLE, hasLock=false; OpenRegionProcedure d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 because future has completed 2024-12-06T16:02:34,148 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-06T16:02:34,148 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; OpenRegionProcedure d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 in 171 msec 2024-12-06T16:02:34,150 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-12-06T16:02:34,150 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d252d31b6d43033dcd4545dc84403a2e, ASSIGN in 326 msec 2024-12-06T16:02:34,150 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T16:02:34,151 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733500954150"}]},"ts":"1733500954150"} 2024-12-06T16:02:34,152 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-06T16:02:34,152 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T16:02:34,154 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 750 msec 2024-12-06T16:02:34,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-12-06T16:02:34,542 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:02:34,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47e1a424, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,544 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:02:34,544 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:02:34,545 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:02:34,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:02:34,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:02:34,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b0609da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:02:34,546 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:02:34,546 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:34,547 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:02:34,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25ad3ab6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,547 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:02:34,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:02:34,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:02:34,549 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33866, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:02:34,550 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T16:02:34,550 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 85bef17d9292,45907,1733500826847 2024-12-06T16:02:34,550 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7a781fc0 2024-12-06T16:02:34,550 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T16:02:34,551 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42680, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T16:02:34,552 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@407ebc73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,552 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:02:34,552 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:02:34,553 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:02:34,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:02:34,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:02:34,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3733c0ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:02:34,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:02:34,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:34,554 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42698, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:02:34,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ed0ff66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,555 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5159e3f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,555 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:02:34,555 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:02:34,556 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:02:34,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:02:34,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:02:34,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2be66aae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:02:34,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:02:34,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:34,557 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:02:34,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61358907, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,558 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1062bfdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,558 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:02:34,558 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:02:34,559 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:02:34,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:02:34,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:02:34,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66ef9190, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:02:34,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:02:34,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:34,560 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42726, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:02:34,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5448d35f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,561 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d7232e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,561 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:02:34,561 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:02:34,561 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:02:34,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:02:34,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:02:34,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b4ac857, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:02:34,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:02:34,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:34,563 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42740, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:02:34,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd9ea2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,563 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39880b68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,563 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:02:34,564 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:02:34,564 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:02:34,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:02:34,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:02:34,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6affdd59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:02:34,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:02:34,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:34,565 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42754, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:02:34,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7523b669, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,566 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb46e8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,566 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:02:34,566 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:02:34,567 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:02:34,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:02:34,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:02:34,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51992634, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:02:34,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:02:34,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:34,568 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:02:34,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ae20b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,569 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1593fd05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,569 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:02:34,569 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:02:34,570 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:02:34,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:02:34,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:02:34,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45af9973, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:02:34,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:02:34,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:34,571 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42794, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:02:34,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bf8ac25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,572 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@729114ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,572 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:02:34,572 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:02:34,573 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:02:34,573 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:02:34,573 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:02:34,573 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19fab789, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,573 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:02:34,573 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:02:34,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:34,574 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:02:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c566de7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,575 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@139f4c00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,575 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:02:34,575 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:02:34,576 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:02:34,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:02:34,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:02:34,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dd23b25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:02:34,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:02:34,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:34,577 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42824, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:02:34,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b87c6ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,578 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14b03bcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,578 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:02:34,578 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:02:34,579 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:02:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:02:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:02:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a426882, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:02:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:02:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:34,580 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42844, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:02:34,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ae9b5eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:34,581 DEBUG [Thread-2097 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:02:34,581 DEBUG [Thread-2116 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:02:34,581 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:02:34,581 DEBUG [Thread-2101 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:02:34,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:02:34,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:02:34,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:02:34,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:02:34,582 DEBUG [Thread-2108 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:02:34,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-12-06T16:02:34,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:02:34,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:02:34,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:02:34,583 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:02:34,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-06T16:02:34,583 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:02:34,583 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:02:34,584 DEBUG [Thread-2103 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:02:34,584 DEBUG [Thread-2110 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:02:34,584 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:02:34,584 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:02:34,584 DEBUG [Thread-2099 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:02:34,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:02:34,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:02:34,585 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:02:34,585 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33900, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:02:34,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2] 2024-12-06T16:02:34,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:02:34,585 DEBUG [Thread-2114 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:02:34,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:02:34,586 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33912, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:02:34,586 DEBUG [Thread-2105 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:02:34,586 DEBUG [Thread-2114 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:02:34,586 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:02:34,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:02:34,587 DEBUG [Thread-2114 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:02:34,587 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:02:34,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:02:34,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2] 2024-12-06T16:02:34,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:02:34,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2] 2024-12-06T16:02:34,587 DEBUG [Thread-2112 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:02:34,588 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:02:34,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2] 2024-12-06T16:02:34,588 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33926, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:02:34,589 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33934, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:02:34,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:02:34,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:02:34,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2] 2024-12-06T16:02:34,590 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:02:34,591 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33948, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:02:34,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2] 2024-12-06T16:02:34,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2] 2024-12-06T16:02:34,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:34,596 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:02:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2] 2024-12-06T16:02:34,596 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d252d31b6d43033dcd4545dc84403a2e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-06T16:02:34,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2] 2024-12-06T16:02:34,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2] 2024-12-06T16:02:34,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/336febd4ab0748f1837d4307d7071176 is 50, key is test_row_0/A:col10/1733500954593/Put/seqid=0 2024-12-06T16:02:34,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742295_1471 (size=12001) 2024-12-06T16:02:34,624 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/336febd4ab0748f1837d4307d7071176 2024-12-06T16:02:34,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/f882d547a3484cfc946739ec9afc6a9d is 50, key is test_row_0/B:col10/1733500954593/Put/seqid=0 2024-12-06T16:02:34,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742296_1472 (size=12001) 2024-12-06T16:02:34,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501014662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:34,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501014662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:34,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:34,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:34,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:34,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501014664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:34,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:34,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:34,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:34,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:34,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:34,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:34,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501014664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:34,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:34,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:34,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:34,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-06T16:02:34,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501014679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:34,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-06T16:02:34,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:34,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:34,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:34,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:34,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:34,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:34,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501014781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:34,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:34,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:34,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:34,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501014792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:34,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:34,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501014792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:34,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:34,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:34,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501014792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:34,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:34,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:34,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:34,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:34,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:34,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:34,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501014802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:34,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-06T16:02:34,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:34,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:34,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:34,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:34,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:34,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-06T16:02:35,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:35,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501014992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:35,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:35,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:35,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501015011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:35,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:35,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501015011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:35,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:35,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:35,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:35,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501015011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:35,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:35,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:35,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501015022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:35,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:35,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-06T16:02:35,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:35,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:35,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:35,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:35,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:35,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:35,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/f882d547a3484cfc946739ec9afc6a9d 2024-12-06T16:02:35,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/489acfa10f8d444cb6d63e348544f442 is 50, key is test_row_0/C:col10/1733500954593/Put/seqid=0 2024-12-06T16:02:35,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742297_1473 (size=12001) 2024-12-06T16:02:35,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/489acfa10f8d444cb6d63e348544f442 2024-12-06T16:02:35,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/336febd4ab0748f1837d4307d7071176 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/336febd4ab0748f1837d4307d7071176 2024-12-06T16:02:35,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/336febd4ab0748f1837d4307d7071176, entries=150, sequenceid=12, filesize=11.7 K 2024-12-06T16:02:35,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/f882d547a3484cfc946739ec9afc6a9d as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/f882d547a3484cfc946739ec9afc6a9d 2024-12-06T16:02:35,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/f882d547a3484cfc946739ec9afc6a9d, entries=150, sequenceid=12, filesize=11.7 K 2024-12-06T16:02:35,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/489acfa10f8d444cb6d63e348544f442 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/489acfa10f8d444cb6d63e348544f442 2024-12-06T16:02:35,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/489acfa10f8d444cb6d63e348544f442, entries=150, sequenceid=12, filesize=11.7 K 2024-12-06T16:02:35,106 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for d252d31b6d43033dcd4545dc84403a2e in 510ms, sequenceid=12, compaction requested=false 2024-12-06T16:02:35,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:35,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-06T16:02:35,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:35,203 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing d252d31b6d43033dcd4545dc84403a2e 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-06T16:02:35,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/57a4102a08674c7c8b04c64c5aef2667 is 50, key is test_row_0/A:col10/1733500954662/Put/seqid=0 2024-12-06T16:02:35,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-06T16:02:35,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742298_1474 (size=12001) 2024-12-06T16:02:35,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:35,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:35,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:35,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501015335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:35,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:35,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:35,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501015339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:35,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:35,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:35,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501015341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:35,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:35,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:35,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501015346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:35,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:35,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501015346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:35,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:35,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:35,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:35,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501015452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:35,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501015452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:35,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501015452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:35,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:35,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501015453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:35,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:35,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501015453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:35,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:35,614 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/57a4102a08674c7c8b04c64c5aef2667 2024-12-06T16:02:35,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/a411fb3527be445ba11ce0d7646b0343 is 50, key is test_row_0/B:col10/1733500954662/Put/seqid=0 2024-12-06T16:02:35,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742299_1475 (size=12001) 2024-12-06T16:02:35,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501015672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:35,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:35,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501015672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:35,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:35,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:35,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501015672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:35,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:35,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501015672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:35,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:35,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:35,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501015673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:35,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:35,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-06T16:02:35,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501015992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:35,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501015992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:35,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:35,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501015992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:35,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:35,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501015993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:35,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:35,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:35,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501015993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:35,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:35,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:35,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:35,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:35,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:36,030 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/a411fb3527be445ba11ce0d7646b0343 2024-12-06T16:02:36,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/11c9d341002d4be1bde5f70c64b9cb3e is 50, key is test_row_0/C:col10/1733500954662/Put/seqid=0 2024-12-06T16:02:36,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742300_1476 (size=12001) 2024-12-06T16:02:36,045 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/11c9d341002d4be1bde5f70c64b9cb3e 2024-12-06T16:02:36,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/57a4102a08674c7c8b04c64c5aef2667 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/57a4102a08674c7c8b04c64c5aef2667 2024-12-06T16:02:36,053 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/57a4102a08674c7c8b04c64c5aef2667, entries=150, sequenceid=37, filesize=11.7 K 2024-12-06T16:02:36,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/a411fb3527be445ba11ce0d7646b0343 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/a411fb3527be445ba11ce0d7646b0343 2024-12-06T16:02:36,057 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/a411fb3527be445ba11ce0d7646b0343, entries=150, sequenceid=37, filesize=11.7 K 2024-12-06T16:02:36,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/11c9d341002d4be1bde5f70c64b9cb3e as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/11c9d341002d4be1bde5f70c64b9cb3e 2024-12-06T16:02:36,066 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/11c9d341002d4be1bde5f70c64b9cb3e, entries=150, sequenceid=37, filesize=11.7 K 2024-12-06T16:02:36,067 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for d252d31b6d43033dcd4545dc84403a2e in 864ms, sequenceid=37, compaction requested=false 2024-12-06T16:02:36,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:36,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:36,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-12-06T16:02:36,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-12-06T16:02:36,071 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-06T16:02:36,071 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4850 sec 2024-12-06T16:02:36,073 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.4910 sec 2024-12-06T16:02:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:36,507 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:02:36,507 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d252d31b6d43033dcd4545dc84403a2e 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-06T16:02:36,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/462d6a05a5f841f3a26154c9b500fb50 is 50, key is test_row_0/A:col10/1733500956506/Put/seqid=0 2024-12-06T16:02:36,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742301_1477 (size=12001) 2024-12-06T16:02:36,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:36,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501016563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:36,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:36,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:36,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:36,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:36,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501016565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:36,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:36,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501016566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:36,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:36,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:36,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:36,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:36,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:36,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:36,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:36,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501016567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:36,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:36,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:36,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:36,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501016569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:36,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:36,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:36,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:36,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:36,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:36,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501016681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:36,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:36,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:36,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501016681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:36,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:36,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:36,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:36,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:36,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:36,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501016682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:36,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:36,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:36,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:36,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:36,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:36,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501016692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:36,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:36,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501016692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:36,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-06T16:02:36,742 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:02:36,742 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:02:36,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-12-06T16:02:36,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-06T16:02:36,744 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:02:36,745 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:02:36,745 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:02:36,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-06T16:02:36,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-06T16:02:36,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:36,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:36,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:36,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:36,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:36,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:36,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:36,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501016902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:36,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:36,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501016902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:36,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:36,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501016902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:36,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:36,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:36,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:36,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:36,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:36,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:36,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:36,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:36,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:36,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:36,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501016911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:36,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:36,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501016912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:36,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:36,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:36,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:36,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:36,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:36,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:36,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/462d6a05a5f841f3a26154c9b500fb50 2024-12-06T16:02:36,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/0b235e64ac0c4ec09a60327072a06b03 is 50, key is test_row_0/B:col10/1733500956506/Put/seqid=0 2024-12-06T16:02:36,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742302_1478 (size=12001) 2024-12-06T16:02:37,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-06T16:02:37,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:37,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:37,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:37,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:37,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:37,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:37,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-06T16:02:37,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-06T16:02:37,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:37,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:37,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:37,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:37,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:37,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:37,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501017222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:37,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:37,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:37,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501017222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:37,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:37,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501017222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:37,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:37,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:37,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:37,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:37,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:37,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:37,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:37,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501017232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:37,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:37,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:37,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:37,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:37,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501017232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:37,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:37,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:37,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:37,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/0b235e64ac0c4ec09a60327072a06b03 2024-12-06T16:02:37,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-06T16:02:37,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:37,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:37,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:37,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:37,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:37,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:37,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/f4899af17d6d4659a1898b71927a22f0 is 50, key is test_row_0/C:col10/1733500956506/Put/seqid=0 2024-12-06T16:02:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-06T16:02:37,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742303_1479 (size=12001) 2024-12-06T16:02:37,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/f4899af17d6d4659a1898b71927a22f0 2024-12-06T16:02:37,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/462d6a05a5f841f3a26154c9b500fb50 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/462d6a05a5f841f3a26154c9b500fb50 2024-12-06T16:02:37,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/462d6a05a5f841f3a26154c9b500fb50, entries=150, sequenceid=49, filesize=11.7 K 2024-12-06T16:02:37,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/0b235e64ac0c4ec09a60327072a06b03 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/0b235e64ac0c4ec09a60327072a06b03 2024-12-06T16:02:37,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/0b235e64ac0c4ec09a60327072a06b03, entries=150, sequenceid=49, filesize=11.7 K 2024-12-06T16:02:37,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/f4899af17d6d4659a1898b71927a22f0 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/f4899af17d6d4659a1898b71927a22f0 2024-12-06T16:02:37,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/f4899af17d6d4659a1898b71927a22f0, entries=150, sequenceid=49, filesize=11.7 K 2024-12-06T16:02:37,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for d252d31b6d43033dcd4545dc84403a2e in 893ms, sequenceid=49, compaction requested=true 2024-12-06T16:02:37,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:37,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d252d31b6d43033dcd4545dc84403a2e:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:02:37,400 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:37,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:37,400 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:37,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d252d31b6d43033dcd4545dc84403a2e:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:02:37,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:37,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d252d31b6d43033dcd4545dc84403a2e:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:02:37,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:37,401 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:37,401 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:37,401 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): d252d31b6d43033dcd4545dc84403a2e/B is initiating minor compaction (all files) 2024-12-06T16:02:37,401 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): d252d31b6d43033dcd4545dc84403a2e/A is initiating minor compaction (all files) 2024-12-06T16:02:37,401 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d252d31b6d43033dcd4545dc84403a2e/B in TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:37,401 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d252d31b6d43033dcd4545dc84403a2e/A in TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:37,401 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/f882d547a3484cfc946739ec9afc6a9d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/a411fb3527be445ba11ce0d7646b0343, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/0b235e64ac0c4ec09a60327072a06b03] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp, totalSize=35.2 K 2024-12-06T16:02:37,401 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/336febd4ab0748f1837d4307d7071176, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/57a4102a08674c7c8b04c64c5aef2667, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/462d6a05a5f841f3a26154c9b500fb50] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp, totalSize=35.2 K 2024-12-06T16:02:37,402 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 336febd4ab0748f1837d4307d7071176, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733500954592 2024-12-06T16:02:37,402 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting f882d547a3484cfc946739ec9afc6a9d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733500954592 2024-12-06T16:02:37,402 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57a4102a08674c7c8b04c64c5aef2667, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1733500954627 2024-12-06T16:02:37,402 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting a411fb3527be445ba11ce0d7646b0343, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1733500954627 2024-12-06T16:02:37,402 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 462d6a05a5f841f3a26154c9b500fb50, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733500955338 2024-12-06T16:02:37,402 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b235e64ac0c4ec09a60327072a06b03, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733500955338 2024-12-06T16:02:37,412 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d252d31b6d43033dcd4545dc84403a2e#A#compaction#402 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:37,413 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d252d31b6d43033dcd4545dc84403a2e#B#compaction#403 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:37,413 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/abb942014ea0493e9f34dfc309fafb43 is 50, key is test_row_0/A:col10/1733500956506/Put/seqid=0 2024-12-06T16:02:37,413 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/55137a9b2d24497688e4d5798bf8e107 is 50, key is test_row_0/B:col10/1733500956506/Put/seqid=0 2024-12-06T16:02:37,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742304_1480 (size=12104) 2024-12-06T16:02:37,432 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/55137a9b2d24497688e4d5798bf8e107 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/55137a9b2d24497688e4d5798bf8e107 2024-12-06T16:02:37,440 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d252d31b6d43033dcd4545dc84403a2e/B of d252d31b6d43033dcd4545dc84403a2e into 55137a9b2d24497688e4d5798bf8e107(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:37,440 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:37,440 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., storeName=d252d31b6d43033dcd4545dc84403a2e/B, priority=13, startTime=1733500957400; duration=0sec 2024-12-06T16:02:37,440 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:37,440 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d252d31b6d43033dcd4545dc84403a2e:B 2024-12-06T16:02:37,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742305_1481 (size=12104) 2024-12-06T16:02:37,441 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:37,441 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:37,441 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): d252d31b6d43033dcd4545dc84403a2e/C is initiating minor compaction (all files) 2024-12-06T16:02:37,441 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d252d31b6d43033dcd4545dc84403a2e/C in TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:37,441 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/489acfa10f8d444cb6d63e348544f442, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/11c9d341002d4be1bde5f70c64b9cb3e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/f4899af17d6d4659a1898b71927a22f0] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp, totalSize=35.2 K 2024-12-06T16:02:37,442 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 489acfa10f8d444cb6d63e348544f442, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733500954592 2024-12-06T16:02:37,442 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 11c9d341002d4be1bde5f70c64b9cb3e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1733500954627 2024-12-06T16:02:37,442 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting f4899af17d6d4659a1898b71927a22f0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733500955338 2024-12-06T16:02:37,457 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d252d31b6d43033dcd4545dc84403a2e#C#compaction#404 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:37,457 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/385ca8459c6f468fbe6369db0febe726 is 50, key is test_row_0/C:col10/1733500956506/Put/seqid=0 2024-12-06T16:02:37,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742306_1482 (size=12104) 2024-12-06T16:02:37,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-06T16:02:37,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:37,510 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing d252d31b6d43033dcd4545dc84403a2e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-06T16:02:37,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/0a9928b2041043cbaa50865de24ac4a2 is 50, key is test_row_0/A:col10/1733500956567/Put/seqid=0 2024-12-06T16:02:37,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742307_1483 (size=12001) 2024-12-06T16:02:37,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:37,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:37,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:37,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501017756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:37,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:37,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:37,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501017757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:37,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:37,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:37,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:37,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:37,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501017763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:37,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:37,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:37,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:37,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:37,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:37,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:37,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501017765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:37,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:37,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501017766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:37,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:37,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:37,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:37,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:37,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:37,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:37,845 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/abb942014ea0493e9f34dfc309fafb43 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/abb942014ea0493e9f34dfc309fafb43 2024-12-06T16:02:37,851 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d252d31b6d43033dcd4545dc84403a2e/A of d252d31b6d43033dcd4545dc84403a2e into abb942014ea0493e9f34dfc309fafb43(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:37,851 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:37,851 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., storeName=d252d31b6d43033dcd4545dc84403a2e/A, priority=13, startTime=1733500957400; duration=0sec 2024-12-06T16:02:37,852 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:37,852 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d252d31b6d43033dcd4545dc84403a2e:A 2024-12-06T16:02:37,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:37,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501017872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:37,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:37,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:37,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:37,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:37,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501017872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:37,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-06T16:02:37,886 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/385ca8459c6f468fbe6369db0febe726 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/385ca8459c6f468fbe6369db0febe726 2024-12-06T16:02:37,890 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d252d31b6d43033dcd4545dc84403a2e/C of d252d31b6d43033dcd4545dc84403a2e into 385ca8459c6f468fbe6369db0febe726(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:37,890 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:37,890 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., storeName=d252d31b6d43033dcd4545dc84403a2e/C, priority=13, startTime=1733500957401; duration=0sec 2024-12-06T16:02:37,890 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:37,890 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d252d31b6d43033dcd4545dc84403a2e:C 2024-12-06T16:02:37,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:37,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501017883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:37,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:37,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:37,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:37,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:37,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501017884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:37,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:37,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501017884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:37,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:37,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:37,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:37,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:37,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:37,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:37,929 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/0a9928b2041043cbaa50865de24ac4a2 2024-12-06T16:02:37,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/1d9e87cc8fcb4c7588d5b9b890cbea9a is 50, key is test_row_0/B:col10/1733500956567/Put/seqid=0 2024-12-06T16:02:37,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742308_1484 (size=12001) 2024-12-06T16:02:37,944 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/1d9e87cc8fcb4c7588d5b9b890cbea9a 2024-12-06T16:02:37,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/bbcd88ea25124715a6300b819c8fad7a is 50, key is test_row_0/C:col10/1733500956567/Put/seqid=0 2024-12-06T16:02:37,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742309_1485 (size=12001) 2024-12-06T16:02:38,071 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T16:02:38,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:38,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501018091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:38,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:38,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501018092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:38,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:38,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:38,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:38,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501018103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:38,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:38,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:38,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501018103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:38,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:38,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501018103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:38,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:38,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:38,358 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/bbcd88ea25124715a6300b819c8fad7a 2024-12-06T16:02:38,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/0a9928b2041043cbaa50865de24ac4a2 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/0a9928b2041043cbaa50865de24ac4a2 2024-12-06T16:02:38,366 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/0a9928b2041043cbaa50865de24ac4a2, entries=150, sequenceid=74, filesize=11.7 K 2024-12-06T16:02:38,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/1d9e87cc8fcb4c7588d5b9b890cbea9a as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/1d9e87cc8fcb4c7588d5b9b890cbea9a 2024-12-06T16:02:38,370 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/1d9e87cc8fcb4c7588d5b9b890cbea9a, entries=150, sequenceid=74, filesize=11.7 K 2024-12-06T16:02:38,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/bbcd88ea25124715a6300b819c8fad7a as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/bbcd88ea25124715a6300b819c8fad7a 2024-12-06T16:02:38,375 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/bbcd88ea25124715a6300b819c8fad7a, entries=150, sequenceid=74, filesize=11.7 K 2024-12-06T16:02:38,375 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for d252d31b6d43033dcd4545dc84403a2e in 865ms, sequenceid=74, compaction requested=false 2024-12-06T16:02:38,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:38,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:38,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-12-06T16:02:38,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-12-06T16:02:38,382 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-06T16:02:38,382 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6320 sec 2024-12-06T16:02:38,385 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.6410 sec 2024-12-06T16:02:38,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:38,406 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:02:38,406 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d252d31b6d43033dcd4545dc84403a2e 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-06T16:02:38,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/81208f65481a49978209dd64c6b94b6f is 50, key is test_row_0/A:col10/1733500957753/Put/seqid=0 2024-12-06T16:02:38,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742310_1486 (size=14341) 2024-12-06T16:02:38,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:38,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501018455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:38,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:38,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501018456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:38,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:38,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501018458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:38,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:38,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:38,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:38,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:38,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501018464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:38,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:38,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501018465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:38,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:38,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:38,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:38,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501018582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:38,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:38,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501018582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:38,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:38,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:38,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501018583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:38,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:38,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:38,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501018583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:38,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:38,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:38,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:38,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501018583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:38,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:38,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501018802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:38,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:38,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501018802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:38,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:38,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501018802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:38,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:38,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:38,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:38,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501018802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:38,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:38,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:38,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501018803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:38,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:38,814 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,814 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:38,814 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:38,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/81208f65481a49978209dd64c6b94b6f 2024-12-06T16:02:38,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/18514aac7aa9424aae04909b921e2e67 is 50, key is test_row_0/B:col10/1733500957753/Put/seqid=0 2024-12-06T16:02:38,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742311_1487 (size=12001) 2024-12-06T16:02:38,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-06T16:02:38,892 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:02:38,892 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:02:38,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-12-06T16:02:38,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-06T16:02:38,895 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:02:38,896 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:02:38,896 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:02:39,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-06T16:02:39,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-06T16:02:39,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:39,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:39,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:39,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:39,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:39,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:39,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:39,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501019121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:39,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:39,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501019122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:39,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:39,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501019122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:39,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:39,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:39,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:39,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501019122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:39,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:39,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:39,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501019122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:39,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:39,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:39,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-06T16:02:39,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:39,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:39,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:39,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:39,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:39,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-06T16:02:39,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/18514aac7aa9424aae04909b921e2e67 2024-12-06T16:02:39,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/3e54f48a58c54201989dd183014d7b55 is 50, key is test_row_0/C:col10/1733500957753/Put/seqid=0 2024-12-06T16:02:39,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742312_1488 (size=12001) 2024-12-06T16:02:39,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/3e54f48a58c54201989dd183014d7b55 2024-12-06T16:02:39,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/81208f65481a49978209dd64c6b94b6f as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/81208f65481a49978209dd64c6b94b6f 2024-12-06T16:02:39,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/81208f65481a49978209dd64c6b94b6f, entries=200, sequenceid=89, filesize=14.0 K 2024-12-06T16:02:39,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/18514aac7aa9424aae04909b921e2e67 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/18514aac7aa9424aae04909b921e2e67 2024-12-06T16:02:39,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/18514aac7aa9424aae04909b921e2e67, entries=150, sequenceid=89, filesize=11.7 K 2024-12-06T16:02:39,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/3e54f48a58c54201989dd183014d7b55 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/3e54f48a58c54201989dd183014d7b55 2024-12-06T16:02:39,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/3e54f48a58c54201989dd183014d7b55, entries=150, sequenceid=89, filesize=11.7 K 2024-12-06T16:02:39,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for d252d31b6d43033dcd4545dc84403a2e in 898ms, sequenceid=89, compaction requested=true 2024-12-06T16:02:39,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:39,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d252d31b6d43033dcd4545dc84403a2e:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:02:39,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:39,304 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:39,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d252d31b6d43033dcd4545dc84403a2e:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:02:39,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:39,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d252d31b6d43033dcd4545dc84403a2e:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:02:39,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-06T16:02:39,305 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:39,305 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:39,305 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): d252d31b6d43033dcd4545dc84403a2e/A is initiating minor compaction (all files) 2024-12-06T16:02:39,305 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d252d31b6d43033dcd4545dc84403a2e/A in TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:39,306 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/abb942014ea0493e9f34dfc309fafb43, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/0a9928b2041043cbaa50865de24ac4a2, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/81208f65481a49978209dd64c6b94b6f] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp, totalSize=37.5 K 2024-12-06T16:02:39,306 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:39,306 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting abb942014ea0493e9f34dfc309fafb43, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733500955338 2024-12-06T16:02:39,306 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): d252d31b6d43033dcd4545dc84403a2e/B is initiating minor compaction (all files) 2024-12-06T16:02:39,306 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d252d31b6d43033dcd4545dc84403a2e/B in TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:39,306 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/55137a9b2d24497688e4d5798bf8e107, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/1d9e87cc8fcb4c7588d5b9b890cbea9a, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/18514aac7aa9424aae04909b921e2e67] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp, totalSize=35.3 K 2024-12-06T16:02:39,306 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a9928b2041043cbaa50865de24ac4a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1733500956563 2024-12-06T16:02:39,307 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81208f65481a49978209dd64c6b94b6f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733500957753 2024-12-06T16:02:39,307 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 55137a9b2d24497688e4d5798bf8e107, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733500955338 2024-12-06T16:02:39,307 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d9e87cc8fcb4c7588d5b9b890cbea9a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1733500956563 2024-12-06T16:02:39,308 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 18514aac7aa9424aae04909b921e2e67, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733500957753 2024-12-06T16:02:39,318 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d252d31b6d43033dcd4545dc84403a2e#A#compaction#411 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:39,318 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/4e270362e27447a4b0a7a11e1dd654d6 is 50, key is test_row_0/A:col10/1733500957753/Put/seqid=0 2024-12-06T16:02:39,321 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d252d31b6d43033dcd4545dc84403a2e#B#compaction#412 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:39,322 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/57f4033b4d7c490b8871ffe974be7efe is 50, key is test_row_0/B:col10/1733500957753/Put/seqid=0 2024-12-06T16:02:39,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742313_1489 (size=12207) 2024-12-06T16:02:39,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742314_1490 (size=12207) 2024-12-06T16:02:39,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-06T16:02:39,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:39,359 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing d252d31b6d43033dcd4545dc84403a2e 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-06T16:02:39,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/2c47771751a9454282b5c96a84c48ae3 is 50, key is test_row_0/A:col10/1733500958452/Put/seqid=0 2024-12-06T16:02:39,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742315_1491 (size=12001) 2024-12-06T16:02:39,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-06T16:02:39,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:39,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:39,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:39,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501019645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:39,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:39,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:39,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501019656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:39,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:39,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:39,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501019659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:39,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:39,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:39,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501019660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:39,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:39,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:39,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501019661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:39,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:39,738 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/4e270362e27447a4b0a7a11e1dd654d6 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/4e270362e27447a4b0a7a11e1dd654d6 2024-12-06T16:02:39,742 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d252d31b6d43033dcd4545dc84403a2e/A of d252d31b6d43033dcd4545dc84403a2e into 4e270362e27447a4b0a7a11e1dd654d6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:39,742 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:39,742 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., storeName=d252d31b6d43033dcd4545dc84403a2e/A, priority=13, startTime=1733500959304; duration=0sec 2024-12-06T16:02:39,743 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:39,743 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d252d31b6d43033dcd4545dc84403a2e:A 2024-12-06T16:02:39,743 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:39,744 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:39,744 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): d252d31b6d43033dcd4545dc84403a2e/C is initiating minor compaction (all files) 2024-12-06T16:02:39,744 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d252d31b6d43033dcd4545dc84403a2e/C in TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:39,744 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/385ca8459c6f468fbe6369db0febe726, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/bbcd88ea25124715a6300b819c8fad7a, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/3e54f48a58c54201989dd183014d7b55] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp, totalSize=35.3 K 2024-12-06T16:02:39,745 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 385ca8459c6f468fbe6369db0febe726, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733500955338 2024-12-06T16:02:39,745 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbcd88ea25124715a6300b819c8fad7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1733500956563 2024-12-06T16:02:39,745 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e54f48a58c54201989dd183014d7b55, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733500957753 2024-12-06T16:02:39,747 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/57f4033b4d7c490b8871ffe974be7efe as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/57f4033b4d7c490b8871ffe974be7efe 2024-12-06T16:02:39,751 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d252d31b6d43033dcd4545dc84403a2e/B of d252d31b6d43033dcd4545dc84403a2e into 57f4033b4d7c490b8871ffe974be7efe(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:39,751 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:39,751 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., storeName=d252d31b6d43033dcd4545dc84403a2e/B, priority=13, startTime=1733500959304; duration=0sec 2024-12-06T16:02:39,751 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:39,751 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d252d31b6d43033dcd4545dc84403a2e:B 2024-12-06T16:02:39,752 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d252d31b6d43033dcd4545dc84403a2e#C#compaction#414 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:39,753 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/1a15b8ce0917446481175d150961c876 is 50, key is test_row_0/C:col10/1733500957753/Put/seqid=0 2024-12-06T16:02:39,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742316_1492 (size=12207) 2024-12-06T16:02:39,770 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/2c47771751a9454282b5c96a84c48ae3 2024-12-06T16:02:39,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/4e1f06be0f7b4035a4d7026a60c9eda1 is 50, key is test_row_0/B:col10/1733500958452/Put/seqid=0 2024-12-06T16:02:39,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:39,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501019772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:39,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:39,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501019772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:39,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:39,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:39,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501019782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:39,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:39,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501019782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:39,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:39,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501019782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:39,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:39,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:39,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742317_1493 (size=12001) 2024-12-06T16:02:39,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501019992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:39,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501019992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:39,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:39,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501019992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:39,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:39,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501019993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:39,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:39,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501019993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:39,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:39,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:39,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:39,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-06T16:02:40,162 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/1a15b8ce0917446481175d150961c876 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/1a15b8ce0917446481175d150961c876 2024-12-06T16:02:40,168 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d252d31b6d43033dcd4545dc84403a2e/C of d252d31b6d43033dcd4545dc84403a2e into 1a15b8ce0917446481175d150961c876(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:40,168 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:40,168 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., storeName=d252d31b6d43033dcd4545dc84403a2e/C, priority=13, startTime=1733500959304; duration=0sec 2024-12-06T16:02:40,168 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:40,168 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d252d31b6d43033dcd4545dc84403a2e:C 2024-12-06T16:02:40,195 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/4e1f06be0f7b4035a4d7026a60c9eda1 2024-12-06T16:02:40,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/68c7d6e1611b4a7184eee8acbec66778 is 50, key is test_row_0/C:col10/1733500958452/Put/seqid=0 2024-12-06T16:02:40,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742318_1494 (size=12001) 2024-12-06T16:02:40,207 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/68c7d6e1611b4a7184eee8acbec66778 2024-12-06T16:02:40,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/2c47771751a9454282b5c96a84c48ae3 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/2c47771751a9454282b5c96a84c48ae3 2024-12-06T16:02:40,218 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/2c47771751a9454282b5c96a84c48ae3, entries=150, sequenceid=112, filesize=11.7 K 2024-12-06T16:02:40,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/4e1f06be0f7b4035a4d7026a60c9eda1 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/4e1f06be0f7b4035a4d7026a60c9eda1 2024-12-06T16:02:40,226 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/4e1f06be0f7b4035a4d7026a60c9eda1, entries=150, sequenceid=112, filesize=11.7 K 2024-12-06T16:02:40,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/68c7d6e1611b4a7184eee8acbec66778 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/68c7d6e1611b4a7184eee8acbec66778 2024-12-06T16:02:40,232 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/68c7d6e1611b4a7184eee8acbec66778, entries=150, sequenceid=112, filesize=11.7 K 2024-12-06T16:02:40,233 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for d252d31b6d43033dcd4545dc84403a2e in 874ms, sequenceid=112, compaction requested=false 2024-12-06T16:02:40,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:40,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:40,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-12-06T16:02:40,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-12-06T16:02:40,237 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-06T16:02:40,237 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3390 sec 2024-12-06T16:02:40,244 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.3450 sec 2024-12-06T16:02:40,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:40,307 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:02:40,307 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d252d31b6d43033dcd4545dc84403a2e 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-06T16:02:40,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/675c164f691b49548b349b16ed47e451 is 50, key is test_row_0/A:col10/1733500960305/Put/seqid=0 2024-12-06T16:02:40,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742319_1495 (size=14391) 2024-12-06T16:02:40,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/675c164f691b49548b349b16ed47e451 2024-12-06T16:02:40,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/ee82634d48f8444fa01672e336affba0 is 50, key is test_row_0/B:col10/1733500960305/Put/seqid=0 2024-12-06T16:02:40,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742320_1496 (size=12051) 2024-12-06T16:02:40,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:40,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501020349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:40,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:40,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:40,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:40,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:40,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501020351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:40,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:40,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:40,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:40,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501020352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:40,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:40,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501020354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:40,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:40,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501020363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:40,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:40,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:40,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:40,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:40,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501020472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:40,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:40,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:40,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:40,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:40,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501020472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:40,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:40,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:40,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:40,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:40,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501020472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:40,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:40,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:40,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:40,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:40,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501020472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:40,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:40,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:40,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:40,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:40,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501020482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:40,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:40,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:40,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:40,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:40,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501020692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:40,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:40,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501020692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:40,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:40,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:40,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:40,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501020692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:40,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:40,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:40,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501020692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:40,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:40,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:40,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:40,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:40,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501020702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:40,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:40,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:40,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:40,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/ee82634d48f8444fa01672e336affba0 2024-12-06T16:02:40,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/ddbd6b08c93149799a00de37f514abde is 50, key is test_row_0/C:col10/1733500960305/Put/seqid=0 2024-12-06T16:02:40,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742321_1497 (size=12051) 2024-12-06T16:02:40,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/ddbd6b08c93149799a00de37f514abde 2024-12-06T16:02:40,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/675c164f691b49548b349b16ed47e451 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/675c164f691b49548b349b16ed47e451 2024-12-06T16:02:40,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/675c164f691b49548b349b16ed47e451, entries=200, sequenceid=129, filesize=14.1 K 2024-12-06T16:02:40,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/ee82634d48f8444fa01672e336affba0 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/ee82634d48f8444fa01672e336affba0 2024-12-06T16:02:40,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/ee82634d48f8444fa01672e336affba0, entries=150, sequenceid=129, filesize=11.8 K 2024-12-06T16:02:40,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/ddbd6b08c93149799a00de37f514abde as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/ddbd6b08c93149799a00de37f514abde 2024-12-06T16:02:40,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/ddbd6b08c93149799a00de37f514abde, entries=150, sequenceid=129, filesize=11.8 K 2024-12-06T16:02:40,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for d252d31b6d43033dcd4545dc84403a2e in 454ms, sequenceid=129, compaction requested=true 2024-12-06T16:02:40,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:40,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d252d31b6d43033dcd4545dc84403a2e:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:02:40,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:40,761 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:40,762 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:40,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d252d31b6d43033dcd4545dc84403a2e:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:02:40,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:40,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d252d31b6d43033dcd4545dc84403a2e:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:02:40,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:40,763 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:40,763 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38599 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:40,763 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): d252d31b6d43033dcd4545dc84403a2e/B is initiating minor compaction (all files) 2024-12-06T16:02:40,763 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): d252d31b6d43033dcd4545dc84403a2e/A is initiating minor compaction (all files) 2024-12-06T16:02:40,763 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d252d31b6d43033dcd4545dc84403a2e/B in TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:40,763 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d252d31b6d43033dcd4545dc84403a2e/A in TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:40,763 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/57f4033b4d7c490b8871ffe974be7efe, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/4e1f06be0f7b4035a4d7026a60c9eda1, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/ee82634d48f8444fa01672e336affba0] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp, totalSize=35.4 K 2024-12-06T16:02:40,763 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/4e270362e27447a4b0a7a11e1dd654d6, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/2c47771751a9454282b5c96a84c48ae3, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/675c164f691b49548b349b16ed47e451] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp, totalSize=37.7 K 2024-12-06T16:02:40,763 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 57f4033b4d7c490b8871ffe974be7efe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733500957753 2024-12-06T16:02:40,763 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e270362e27447a4b0a7a11e1dd654d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733500957753 2024-12-06T16:02:40,764 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e1f06be0f7b4035a4d7026a60c9eda1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733500958452 2024-12-06T16:02:40,764 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c47771751a9454282b5c96a84c48ae3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733500958452 2024-12-06T16:02:40,764 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting ee82634d48f8444fa01672e336affba0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733500959643 2024-12-06T16:02:40,764 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 675c164f691b49548b349b16ed47e451, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733500959643 2024-12-06T16:02:40,777 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d252d31b6d43033dcd4545dc84403a2e#A#compaction#421 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:40,777 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/55e61403ca7c4899af0052694fd2348e is 50, key is test_row_0/A:col10/1733500960305/Put/seqid=0 2024-12-06T16:02:40,777 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d252d31b6d43033dcd4545dc84403a2e#B#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:40,778 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/d733c5d620b34f1d8bf5860bb0b03041 is 50, key is test_row_0/B:col10/1733500960305/Put/seqid=0 2024-12-06T16:02:40,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742322_1498 (size=12359) 2024-12-06T16:02:40,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742323_1499 (size=12359) 2024-12-06T16:02:40,805 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/55e61403ca7c4899af0052694fd2348e as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/55e61403ca7c4899af0052694fd2348e 2024-12-06T16:02:40,812 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d252d31b6d43033dcd4545dc84403a2e/A of d252d31b6d43033dcd4545dc84403a2e into 55e61403ca7c4899af0052694fd2348e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:40,812 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:40,812 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., storeName=d252d31b6d43033dcd4545dc84403a2e/A, priority=13, startTime=1733500960761; duration=0sec 2024-12-06T16:02:40,812 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:40,812 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d252d31b6d43033dcd4545dc84403a2e:A 2024-12-06T16:02:40,813 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:40,813 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:40,813 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): d252d31b6d43033dcd4545dc84403a2e/C is initiating minor compaction (all files) 2024-12-06T16:02:40,813 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d252d31b6d43033dcd4545dc84403a2e/C in TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:40,813 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/1a15b8ce0917446481175d150961c876, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/68c7d6e1611b4a7184eee8acbec66778, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/ddbd6b08c93149799a00de37f514abde] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp, totalSize=35.4 K 2024-12-06T16:02:40,814 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a15b8ce0917446481175d150961c876, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733500957753 2024-12-06T16:02:40,814 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68c7d6e1611b4a7184eee8acbec66778, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733500958452 2024-12-06T16:02:40,814 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddbd6b08c93149799a00de37f514abde, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733500959643 2024-12-06T16:02:40,829 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d252d31b6d43033dcd4545dc84403a2e#C#compaction#422 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:40,830 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/ffdf07e7d1534c35b9b5fccaff3cef09 is 50, key is test_row_0/C:col10/1733500960305/Put/seqid=0 2024-12-06T16:02:40,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742324_1500 (size=12359) 2024-12-06T16:02:40,844 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/ffdf07e7d1534c35b9b5fccaff3cef09 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/ffdf07e7d1534c35b9b5fccaff3cef09 2024-12-06T16:02:40,851 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d252d31b6d43033dcd4545dc84403a2e/C of d252d31b6d43033dcd4545dc84403a2e into ffdf07e7d1534c35b9b5fccaff3cef09(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:40,851 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:40,852 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., storeName=d252d31b6d43033dcd4545dc84403a2e/C, priority=13, startTime=1733500960762; duration=0sec 2024-12-06T16:02:40,852 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:40,852 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d252d31b6d43033dcd4545dc84403a2e:C 2024-12-06T16:02:41,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:41,019 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:02:41,019 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d252d31b6d43033dcd4545dc84403a2e 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-06T16:02:41,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/4bbf4bde630c45bea03b892416ab0d74 is 50, key is test_row_0/A:col10/1733500960352/Put/seqid=0 2024-12-06T16:02:41,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742325_1501 (size=14541) 2024-12-06T16:02:41,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/4bbf4bde630c45bea03b892416ab0d74 2024-12-06T16:02:41,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/d8152a203113426dadfd715a2ce1a414 is 50, key is test_row_0/B:col10/1733500960352/Put/seqid=0 2024-12-06T16:02:41,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-06T16:02:41,042 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:02:41,042 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:02:41,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-12-06T16:02:41,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-06T16:02:41,044 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:02:41,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742326_1502 (size=12151) 2024-12-06T16:02:41,045 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:02:41,045 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:02:41,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/d8152a203113426dadfd715a2ce1a414 2024-12-06T16:02:41,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:41,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501021037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:41,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:41,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:41,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501021037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:41,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/4b9253fbbcb7413eafe0c56266f64aa0 is 50, key is test_row_0/C:col10/1733500960352/Put/seqid=0 2024-12-06T16:02:41,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:41,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501021039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:41,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:41,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:41,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:41,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501021042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:41,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:41,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742327_1503 (size=12151) 2024-12-06T16:02:41,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:41,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501021056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:41,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:41,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-06T16:02:41,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:41,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501021162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:41,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:41,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501021162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:41,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:41,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501021162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:41,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:41,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:41,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:41,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501021162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:41,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:41,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:41,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:41,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501021171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:41,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:41,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-06T16:02:41,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:41,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:41,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:41,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:41,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:41,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:41,204 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/d733c5d620b34f1d8bf5860bb0b03041 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/d733c5d620b34f1d8bf5860bb0b03041 2024-12-06T16:02:41,211 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d252d31b6d43033dcd4545dc84403a2e/B of d252d31b6d43033dcd4545dc84403a2e into d733c5d620b34f1d8bf5860bb0b03041(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:41,211 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:41,211 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., storeName=d252d31b6d43033dcd4545dc84403a2e/B, priority=13, startTime=1733500960761; duration=0sec 2024-12-06T16:02:41,211 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:41,211 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d252d31b6d43033dcd4545dc84403a2e:B 2024-12-06T16:02:41,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-06T16:02:41,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:41,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:41,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:41,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:41,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-06T16:02:41,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:41,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501021372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:41,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:41,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:41,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501021372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:41,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:41,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:41,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501021373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:41,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:41,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:41,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501021373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:41,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501021382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:41,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:41,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/4b9253fbbcb7413eafe0c56266f64aa0 2024-12-06T16:02:41,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/4bbf4bde630c45bea03b892416ab0d74 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/4bbf4bde630c45bea03b892416ab0d74 2024-12-06T16:02:41,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/4bbf4bde630c45bea03b892416ab0d74, entries=200, sequenceid=154, filesize=14.2 K 2024-12-06T16:02:41,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/d8152a203113426dadfd715a2ce1a414 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/d8152a203113426dadfd715a2ce1a414 2024-12-06T16:02:41,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/d8152a203113426dadfd715a2ce1a414, entries=150, sequenceid=154, filesize=11.9 K 2024-12-06T16:02:41,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/4b9253fbbcb7413eafe0c56266f64aa0 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/4b9253fbbcb7413eafe0c56266f64aa0 2024-12-06T16:02:41,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/4b9253fbbcb7413eafe0c56266f64aa0, entries=150, sequenceid=154, filesize=11.9 K 2024-12-06T16:02:41,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for d252d31b6d43033dcd4545dc84403a2e in 467ms, sequenceid=154, compaction requested=false 2024-12-06T16:02:41,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:41,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-06T16:02:41,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:41,505 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing d252d31b6d43033dcd4545dc84403a2e 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-06T16:02:41,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/4f36894052064c189cdf735e5ba82d33 is 50, key is test_row_0/A:col10/1733500961040/Put/seqid=0 2024-12-06T16:02:41,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742328_1504 (size=12151) 2024-12-06T16:02:41,518 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/4f36894052064c189cdf735e5ba82d33 2024-12-06T16:02:41,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/9c615b268f154256b3fc73a10c148376 is 50, key is test_row_0/B:col10/1733500961040/Put/seqid=0 2024-12-06T16:02:41,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742329_1505 (size=12151) 2024-12-06T16:02:41,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-06T16:02:41,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:41,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:41,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:41,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501021729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:41,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:41,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:41,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501021730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:41,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:41,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501021735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:41,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:41,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:41,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:41,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501021736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:41,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:41,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501021740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:41,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:41,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:41,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:41,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501021852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:41,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:41,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501021853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:41,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:41,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501021853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:41,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501021853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:41,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501021853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:41,934 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/9c615b268f154256b3fc73a10c148376 2024-12-06T16:02:41,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/6edd7e4e75264a93a3aff43e5f44b32f is 50, key is test_row_0/C:col10/1733500961040/Put/seqid=0 2024-12-06T16:02:41,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742330_1506 (size=12151) 2024-12-06T16:02:42,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:42,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501022072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:42,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:42,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501022072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:42,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:42,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:42,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501022072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:42,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:42,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:42,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:42,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501022072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:42,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:42,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501022072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:42,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:42,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-06T16:02:42,347 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/6edd7e4e75264a93a3aff43e5f44b32f 2024-12-06T16:02:42,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/4f36894052064c189cdf735e5ba82d33 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/4f36894052064c189cdf735e5ba82d33 2024-12-06T16:02:42,362 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/4f36894052064c189cdf735e5ba82d33, entries=150, sequenceid=168, filesize=11.9 K 2024-12-06T16:02:42,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/9c615b268f154256b3fc73a10c148376 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/9c615b268f154256b3fc73a10c148376 2024-12-06T16:02:42,366 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/9c615b268f154256b3fc73a10c148376, entries=150, sequenceid=168, filesize=11.9 K 2024-12-06T16:02:42,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/6edd7e4e75264a93a3aff43e5f44b32f as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/6edd7e4e75264a93a3aff43e5f44b32f 2024-12-06T16:02:42,370 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/6edd7e4e75264a93a3aff43e5f44b32f, entries=150, sequenceid=168, filesize=11.9 K 2024-12-06T16:02:42,371 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for d252d31b6d43033dcd4545dc84403a2e in 867ms, sequenceid=168, compaction requested=true 2024-12-06T16:02:42,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:42,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:42,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-12-06T16:02:42,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-12-06T16:02:42,375 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-06T16:02:42,375 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3280 sec 2024-12-06T16:02:42,377 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.3330 sec 2024-12-06T16:02:42,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:42,400 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:02:42,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d252d31b6d43033dcd4545dc84403a2e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-06T16:02:42,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/750ccae4409b4d80a5fdb1b5f32fcf07 is 50, key is test_row_0/A:col10/1733500961739/Put/seqid=0 2024-12-06T16:02:42,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501022417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:42,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:42,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742331_1507 (size=14541) 2024-12-06T16:02:42,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:42,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501022418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:42,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:42,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501022418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:42,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:42,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:42,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:42,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501022420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:42,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:42,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501022420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:42,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:42,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:42,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:42,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501022522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:42,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:42,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:42,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501022532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:42,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:42,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501022532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:42,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:42,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501022541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:42,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:42,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:42,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501022541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:42,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:42,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:42,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:42,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:42,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501022742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:42,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:42,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:42,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501022761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:42,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:42,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501022761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:42,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501022762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:42,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501022762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:42,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:42,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/750ccae4409b4d80a5fdb1b5f32fcf07 2024-12-06T16:02:42,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/6c3b92b366734bb8a7492a36df19f89e is 50, key is test_row_0/B:col10/1733500961739/Put/seqid=0 2024-12-06T16:02:42,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742332_1508 (size=12151) 2024-12-06T16:02:43,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:43,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501023062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:43,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:43,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:43,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:43,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:43,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501023072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:43,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:43,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:43,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:43,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501023072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:43,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:43,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:43,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:43,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:43,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501023072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:43,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:43,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:43,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:43,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:43,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501023082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:43,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:43,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:43,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:43,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-06T16:02:43,202 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:02:43,203 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:02:43,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-12-06T16:02:43,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-06T16:02:43,205 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:02:43,206 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:02:43,206 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:02:43,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/6c3b92b366734bb8a7492a36df19f89e 2024-12-06T16:02:43,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/ae0767a37c34455fa81f229d1b6a3004 is 50, key is test_row_0/C:col10/1733500961739/Put/seqid=0 2024-12-06T16:02:43,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742333_1509 (size=12151) 2024-12-06T16:02:43,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-06T16:02:43,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-06T16:02:43,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:43,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:43,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:43,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:43,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:43,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:43,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-06T16:02:43,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:43,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:43,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:43,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:43,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:43,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:43,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-06T16:02:43,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:43,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501023581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:43,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:43,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501023592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:43,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:43,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:43,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:43,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501023592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:43,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:43,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:43,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501023592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:43,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:43,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:43,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:43,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:43,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:43,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:43,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:43,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501023611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:43,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:43,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:43,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:43,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/ae0767a37c34455fa81f229d1b6a3004 2024-12-06T16:02:43,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-06T16:02:43,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:43,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:43,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:43,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:43,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:43,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:43,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/750ccae4409b4d80a5fdb1b5f32fcf07 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/750ccae4409b4d80a5fdb1b5f32fcf07 2024-12-06T16:02:43,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/750ccae4409b4d80a5fdb1b5f32fcf07, entries=200, sequenceid=192, filesize=14.2 K 2024-12-06T16:02:43,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/6c3b92b366734bb8a7492a36df19f89e as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/6c3b92b366734bb8a7492a36df19f89e 2024-12-06T16:02:43,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/6c3b92b366734bb8a7492a36df19f89e, entries=150, sequenceid=192, filesize=11.9 K 2024-12-06T16:02:43,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/ae0767a37c34455fa81f229d1b6a3004 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/ae0767a37c34455fa81f229d1b6a3004 2024-12-06T16:02:43,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/ae0767a37c34455fa81f229d1b6a3004, entries=150, sequenceid=192, filesize=11.9 K 2024-12-06T16:02:43,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for d252d31b6d43033dcd4545dc84403a2e in 1281ms, sequenceid=192, compaction requested=true 2024-12-06T16:02:43,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:43,682 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T16:02:43,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d252d31b6d43033dcd4545dc84403a2e:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:02:43,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:43,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d252d31b6d43033dcd4545dc84403a2e:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:02:43,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:43,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d252d31b6d43033dcd4545dc84403a2e:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:02:43,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-06T16:02:43,683 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T16:02:43,685 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53592 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T16:02:43,685 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48812 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T16:02:43,685 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): d252d31b6d43033dcd4545dc84403a2e/A is initiating minor compaction (all files) 2024-12-06T16:02:43,685 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): d252d31b6d43033dcd4545dc84403a2e/B is initiating minor compaction (all files) 2024-12-06T16:02:43,685 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d252d31b6d43033dcd4545dc84403a2e/A in TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:43,685 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d252d31b6d43033dcd4545dc84403a2e/B in TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:43,685 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/d733c5d620b34f1d8bf5860bb0b03041, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/d8152a203113426dadfd715a2ce1a414, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/9c615b268f154256b3fc73a10c148376, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/6c3b92b366734bb8a7492a36df19f89e] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp, totalSize=47.7 K 2024-12-06T16:02:43,685 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/55e61403ca7c4899af0052694fd2348e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/4bbf4bde630c45bea03b892416ab0d74, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/4f36894052064c189cdf735e5ba82d33, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/750ccae4409b4d80a5fdb1b5f32fcf07] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp, totalSize=52.3 K 2024-12-06T16:02:43,685 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting d733c5d620b34f1d8bf5860bb0b03041, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733500959643 2024-12-06T16:02:43,685 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55e61403ca7c4899af0052694fd2348e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733500959643 2024-12-06T16:02:43,686 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bbf4bde630c45bea03b892416ab0d74, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733500960347 2024-12-06T16:02:43,686 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting d8152a203113426dadfd715a2ce1a414, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733500960352 2024-12-06T16:02:43,686 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f36894052064c189cdf735e5ba82d33, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733500961035 2024-12-06T16:02:43,686 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c615b268f154256b3fc73a10c148376, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733500961035 2024-12-06T16:02:43,686 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 750ccae4409b4d80a5fdb1b5f32fcf07, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733500961721 2024-12-06T16:02:43,687 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c3b92b366734bb8a7492a36df19f89e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733500961721 2024-12-06T16:02:43,698 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d252d31b6d43033dcd4545dc84403a2e#A#compaction#432 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:43,698 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d252d31b6d43033dcd4545dc84403a2e#B#compaction#433 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:43,699 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/9c3e588a2317430681205e7329416eb6 is 50, key is test_row_0/A:col10/1733500961739/Put/seqid=0 2024-12-06T16:02:43,699 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/32bf68c2e8994354985ca2145b8baa74 is 50, key is test_row_0/B:col10/1733500961739/Put/seqid=0 2024-12-06T16:02:43,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742334_1510 (size=12595) 2024-12-06T16:02:43,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742335_1511 (size=12595) 2024-12-06T16:02:43,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-06T16:02:43,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:43,819 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing d252d31b6d43033dcd4545dc84403a2e 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-06T16:02:43,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/2dc98ccc9bc646bcabc9d1b03f17674f is 50, key is test_row_0/A:col10/1733500962418/Put/seqid=0 2024-12-06T16:02:43,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-06T16:02:43,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742336_1512 (size=12151) 2024-12-06T16:02:43,834 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/2dc98ccc9bc646bcabc9d1b03f17674f 2024-12-06T16:02:43,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/c4ff191b6ad743639098ffbe242d8411 is 50, key is test_row_0/B:col10/1733500962418/Put/seqid=0 2024-12-06T16:02:43,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742337_1513 (size=12151) 2024-12-06T16:02:44,115 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/9c3e588a2317430681205e7329416eb6 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/9c3e588a2317430681205e7329416eb6 2024-12-06T16:02:44,117 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/32bf68c2e8994354985ca2145b8baa74 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/32bf68c2e8994354985ca2145b8baa74 2024-12-06T16:02:44,123 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d252d31b6d43033dcd4545dc84403a2e/A of d252d31b6d43033dcd4545dc84403a2e into 9c3e588a2317430681205e7329416eb6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:44,123 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:44,123 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., storeName=d252d31b6d43033dcd4545dc84403a2e/A, priority=12, startTime=1733500963682; duration=0sec 2024-12-06T16:02:44,123 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:44,123 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d252d31b6d43033dcd4545dc84403a2e:A 2024-12-06T16:02:44,123 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T16:02:44,125 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48812 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T16:02:44,125 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): d252d31b6d43033dcd4545dc84403a2e/C is initiating minor compaction (all files) 2024-12-06T16:02:44,125 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d252d31b6d43033dcd4545dc84403a2e/C in TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:44,125 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/ffdf07e7d1534c35b9b5fccaff3cef09, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/4b9253fbbcb7413eafe0c56266f64aa0, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/6edd7e4e75264a93a3aff43e5f44b32f, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/ae0767a37c34455fa81f229d1b6a3004] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp, totalSize=47.7 K 2024-12-06T16:02:44,125 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffdf07e7d1534c35b9b5fccaff3cef09, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733500959643 2024-12-06T16:02:44,126 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b9253fbbcb7413eafe0c56266f64aa0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733500960352 2024-12-06T16:02:44,126 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d252d31b6d43033dcd4545dc84403a2e/B of d252d31b6d43033dcd4545dc84403a2e into 32bf68c2e8994354985ca2145b8baa74(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:44,126 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:44,126 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., storeName=d252d31b6d43033dcd4545dc84403a2e/B, priority=12, startTime=1733500963682; duration=0sec 2024-12-06T16:02:44,126 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:44,126 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d252d31b6d43033dcd4545dc84403a2e:B 2024-12-06T16:02:44,126 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6edd7e4e75264a93a3aff43e5f44b32f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733500961035 2024-12-06T16:02:44,127 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae0767a37c34455fa81f229d1b6a3004, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733500961721 2024-12-06T16:02:44,138 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d252d31b6d43033dcd4545dc84403a2e#C#compaction#436 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:44,139 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/f7c3572c60214675be7aaf52df0432f8 is 50, key is test_row_0/C:col10/1733500961739/Put/seqid=0 2024-12-06T16:02:44,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742338_1514 (size=12595) 2024-12-06T16:02:44,160 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/f7c3572c60214675be7aaf52df0432f8 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/f7c3572c60214675be7aaf52df0432f8 2024-12-06T16:02:44,167 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d252d31b6d43033dcd4545dc84403a2e/C of d252d31b6d43033dcd4545dc84403a2e into f7c3572c60214675be7aaf52df0432f8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:44,167 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:44,167 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., storeName=d252d31b6d43033dcd4545dc84403a2e/C, priority=12, startTime=1733500963682; duration=0sec 2024-12-06T16:02:44,167 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:44,167 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d252d31b6d43033dcd4545dc84403a2e:C 2024-12-06T16:02:44,246 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/c4ff191b6ad743639098ffbe242d8411 2024-12-06T16:02:44,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/e811f38c539e46baaabec3c9f2dc2b22 is 50, key is test_row_0/C:col10/1733500962418/Put/seqid=0 2024-12-06T16:02:44,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742339_1515 (size=12151) 2024-12-06T16:02:44,270 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/e811f38c539e46baaabec3c9f2dc2b22 2024-12-06T16:02:44,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/2dc98ccc9bc646bcabc9d1b03f17674f as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/2dc98ccc9bc646bcabc9d1b03f17674f 2024-12-06T16:02:44,283 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/2dc98ccc9bc646bcabc9d1b03f17674f, entries=150, sequenceid=204, filesize=11.9 K 2024-12-06T16:02:44,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/c4ff191b6ad743639098ffbe242d8411 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/c4ff191b6ad743639098ffbe242d8411 2024-12-06T16:02:44,291 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/c4ff191b6ad743639098ffbe242d8411, entries=150, sequenceid=204, filesize=11.9 K 2024-12-06T16:02:44,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/e811f38c539e46baaabec3c9f2dc2b22 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/e811f38c539e46baaabec3c9f2dc2b22 2024-12-06T16:02:44,307 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/e811f38c539e46baaabec3c9f2dc2b22, entries=150, sequenceid=204, filesize=11.9 K 2024-12-06T16:02:44,309 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for d252d31b6d43033dcd4545dc84403a2e in 491ms, sequenceid=204, compaction requested=false 2024-12-06T16:02:44,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:44,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:44,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-12-06T16:02:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-12-06T16:02:44,316 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-06T16:02:44,316 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1060 sec 2024-12-06T16:02:44,319 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.1140 sec 2024-12-06T16:02:44,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-06T16:02:44,343 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:02:44,343 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:02:44,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-12-06T16:02:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-06T16:02:44,346 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:02:44,346 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:02:44,346 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:02:44,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-06T16:02:44,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-06T16:02:44,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:44,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:44,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:44,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-06T16:02:44,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-06T16:02:44,507 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-06T16:02:44,507 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 159 msec 2024-12-06T16:02:44,510 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 165 msec 2024-12-06T16:02:44,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:44,618 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:02:44,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d252d31b6d43033dcd4545dc84403a2e 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-06T16:02:44,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/050f1f6b50a94a6998cf48f6540ac604 is 50, key is test_row_0/A:col10/1733500964617/Put/seqid=0 2024-12-06T16:02:44,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742340_1516 (size=12151) 2024-12-06T16:02:44,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/050f1f6b50a94a6998cf48f6540ac604 2024-12-06T16:02:44,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/d5b4fb72d6a84dc1a872ade5270fdf6d is 50, key is test_row_0/B:col10/1733500964617/Put/seqid=0 2024-12-06T16:02:44,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742341_1517 (size=12151) 2024-12-06T16:02:44,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-06T16:02:44,662 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:02:44,662 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:02:44,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-06T16:02:44,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-06T16:02:44,664 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:02:44,665 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:02:44,665 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:02:44,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:44,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501024678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:44,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:44,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501024681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:44,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:44,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501024683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:44,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501024684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:44,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501024684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:44,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-06T16:02:44,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:44,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501024791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:44,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:44,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:44,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:44,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:44,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501024802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:44,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:44,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501024802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:44,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:44,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501024802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:44,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:44,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:44,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:44,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:44,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501024802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:44,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:44,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:44,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:44,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:44,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:44,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:44,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-06T16:02:44,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:44,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:44,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:44,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:44,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:44,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:44,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-06T16:02:44,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:44,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:44,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:44,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:44,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:44,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:44,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-06T16:02:45,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:45,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501025011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:45,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:45,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:45,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:45,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:45,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501025022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:45,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:45,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:45,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:45,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501025022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:45,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:45,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:45,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:45,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:45,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501025022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:45,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:45,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:45,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501025022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:45,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:45,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:45,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:45,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:45,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:45,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:45,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/d5b4fb72d6a84dc1a872ade5270fdf6d 2024-12-06T16:02:45,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/74ad6c3614a948469b25a1130442c03e is 50, key is test_row_0/C:col10/1733500964617/Put/seqid=0 2024-12-06T16:02:45,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742342_1518 (size=12151) 2024-12-06T16:02:45,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-06T16:02:45,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:45,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:45,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:45,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:45,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:45,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:45,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-06T16:02:45,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:45,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:45,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:45,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:45,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:45,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:45,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-06T16:02:45,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:45,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501025331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:45,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:45,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:45,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:45,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:45,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501025342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:45,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:45,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501025342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:45,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:45,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:45,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:45,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:45,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:45,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501025342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:45,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:45,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:45,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501025342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:45,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-06T16:02:45,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:45,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:45,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:45,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:45,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:45,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:45,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/74ad6c3614a948469b25a1130442c03e 2024-12-06T16:02:45,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/050f1f6b50a94a6998cf48f6540ac604 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/050f1f6b50a94a6998cf48f6540ac604 2024-12-06T16:02:45,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/050f1f6b50a94a6998cf48f6540ac604, entries=150, sequenceid=220, filesize=11.9 K 2024-12-06T16:02:45,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/d5b4fb72d6a84dc1a872ade5270fdf6d as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/d5b4fb72d6a84dc1a872ade5270fdf6d 2024-12-06T16:02:45,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/d5b4fb72d6a84dc1a872ade5270fdf6d, entries=150, sequenceid=220, filesize=11.9 K 2024-12-06T16:02:45,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/74ad6c3614a948469b25a1130442c03e as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/74ad6c3614a948469b25a1130442c03e 2024-12-06T16:02:45,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/74ad6c3614a948469b25a1130442c03e, entries=150, sequenceid=220, filesize=11.9 K 2024-12-06T16:02:45,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for d252d31b6d43033dcd4545dc84403a2e in 872ms, sequenceid=220, compaction requested=true 2024-12-06T16:02:45,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:45,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d252d31b6d43033dcd4545dc84403a2e:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:02:45,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:45,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d252d31b6d43033dcd4545dc84403a2e:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:02:45,491 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:45,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:45,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d252d31b6d43033dcd4545dc84403a2e:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:02:45,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:45,491 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:45,494 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:45,494 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:45,494 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): d252d31b6d43033dcd4545dc84403a2e/A is initiating minor compaction (all files) 2024-12-06T16:02:45,494 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): d252d31b6d43033dcd4545dc84403a2e/B is initiating minor compaction (all files) 2024-12-06T16:02:45,494 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d252d31b6d43033dcd4545dc84403a2e/B in TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:45,494 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d252d31b6d43033dcd4545dc84403a2e/A in TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:45,494 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/32bf68c2e8994354985ca2145b8baa74, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/c4ff191b6ad743639098ffbe242d8411, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/d5b4fb72d6a84dc1a872ade5270fdf6d] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp, totalSize=36.0 K 2024-12-06T16:02:45,494 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/9c3e588a2317430681205e7329416eb6, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/2dc98ccc9bc646bcabc9d1b03f17674f, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/050f1f6b50a94a6998cf48f6540ac604] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp, totalSize=36.0 K 2024-12-06T16:02:45,494 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 32bf68c2e8994354985ca2145b8baa74, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733500961721 2024-12-06T16:02:45,494 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c3e588a2317430681205e7329416eb6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733500961721 2024-12-06T16:02:45,495 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2dc98ccc9bc646bcabc9d1b03f17674f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733500962416 2024-12-06T16:02:45,495 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting c4ff191b6ad743639098ffbe242d8411, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733500962416 2024-12-06T16:02:45,495 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 050f1f6b50a94a6998cf48f6540ac604, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733500964610 2024-12-06T16:02:45,495 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting d5b4fb72d6a84dc1a872ade5270fdf6d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733500964610 2024-12-06T16:02:45,513 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d252d31b6d43033dcd4545dc84403a2e#A#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:45,513 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/94c7898da2a94f12af8a21b48dcc20f2 is 50, key is test_row_0/A:col10/1733500964617/Put/seqid=0 2024-12-06T16:02:45,514 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d252d31b6d43033dcd4545dc84403a2e#B#compaction#442 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:45,514 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/b034e79856534fbeb7b7f866dc7f776d is 50, key is test_row_0/B:col10/1733500964617/Put/seqid=0 2024-12-06T16:02:45,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742344_1520 (size=12697) 2024-12-06T16:02:45,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742343_1519 (size=12697) 2024-12-06T16:02:45,543 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/b034e79856534fbeb7b7f866dc7f776d as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/b034e79856534fbeb7b7f866dc7f776d 2024-12-06T16:02:45,551 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d252d31b6d43033dcd4545dc84403a2e/B of d252d31b6d43033dcd4545dc84403a2e into b034e79856534fbeb7b7f866dc7f776d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:45,551 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:45,551 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., storeName=d252d31b6d43033dcd4545dc84403a2e/B, priority=13, startTime=1733500965491; duration=0sec 2024-12-06T16:02:45,551 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:45,551 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d252d31b6d43033dcd4545dc84403a2e:B 2024-12-06T16:02:45,552 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:45,552 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:45,552 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): d252d31b6d43033dcd4545dc84403a2e/C is initiating minor compaction (all files) 2024-12-06T16:02:45,553 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d252d31b6d43033dcd4545dc84403a2e/C in TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:45,555 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/f7c3572c60214675be7aaf52df0432f8, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/e811f38c539e46baaabec3c9f2dc2b22, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/74ad6c3614a948469b25a1130442c03e] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp, totalSize=36.0 K 2024-12-06T16:02:45,556 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting f7c3572c60214675be7aaf52df0432f8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733500961721 2024-12-06T16:02:45,556 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting e811f38c539e46baaabec3c9f2dc2b22, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733500962416 2024-12-06T16:02:45,557 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 74ad6c3614a948469b25a1130442c03e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733500964610 2024-12-06T16:02:45,568 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d252d31b6d43033dcd4545dc84403a2e#C#compaction#443 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:45,573 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/89cdf2a1301441b9b10f23462fd1ba03 is 50, key is test_row_0/C:col10/1733500964617/Put/seqid=0 2024-12-06T16:02:45,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-06T16:02:45,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:45,586 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing d252d31b6d43033dcd4545dc84403a2e 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-06T16:02:45,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742345_1521 (size=12697) 2024-12-06T16:02:45,606 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/89cdf2a1301441b9b10f23462fd1ba03 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/89cdf2a1301441b9b10f23462fd1ba03 2024-12-06T16:02:45,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/84f341b86b6c46bf95096a34ef48f390 is 50, key is test_row_0/A:col10/1733500964681/Put/seqid=0 2024-12-06T16:02:45,617 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d252d31b6d43033dcd4545dc84403a2e/C of d252d31b6d43033dcd4545dc84403a2e into 89cdf2a1301441b9b10f23462fd1ba03(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:45,617 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:45,617 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., storeName=d252d31b6d43033dcd4545dc84403a2e/C, priority=13, startTime=1733500965491; duration=0sec 2024-12-06T16:02:45,617 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:45,617 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d252d31b6d43033dcd4545dc84403a2e:C 2024-12-06T16:02:45,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742346_1522 (size=12151) 2024-12-06T16:02:45,642 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/84f341b86b6c46bf95096a34ef48f390 2024-12-06T16:02:45,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/18fe10b910e3426b855f7b0d7a4542a9 is 50, key is test_row_0/B:col10/1733500964681/Put/seqid=0 2024-12-06T16:02:45,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742347_1523 (size=12151) 2024-12-06T16:02:45,674 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/18fe10b910e3426b855f7b0d7a4542a9 2024-12-06T16:02:45,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/a4cd53c0c71c4eee826ca251483290b9 is 50, key is test_row_0/C:col10/1733500964681/Put/seqid=0 2024-12-06T16:02:45,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742348_1524 (size=12151) 2024-12-06T16:02:45,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-06T16:02:45,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:45,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:45,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:45,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:45,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501025872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:45,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501025871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:45,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:45,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501025873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:45,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:45,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:45,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:45,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:45,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:45,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:45,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501025885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:45,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:45,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:45,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:45,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:45,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:45,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:45,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501025885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:45,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:45,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:45,944 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/94c7898da2a94f12af8a21b48dcc20f2 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/94c7898da2a94f12af8a21b48dcc20f2 2024-12-06T16:02:45,949 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d252d31b6d43033dcd4545dc84403a2e/A of d252d31b6d43033dcd4545dc84403a2e into 94c7898da2a94f12af8a21b48dcc20f2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:45,949 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:45,949 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., storeName=d252d31b6d43033dcd4545dc84403a2e/A, priority=13, startTime=1733500965490; duration=0sec 2024-12-06T16:02:45,949 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:45,949 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d252d31b6d43033dcd4545dc84403a2e:A 2024-12-06T16:02:46,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:46,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501025992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:46,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:46,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501025992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:46,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:46,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501025992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:46,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:46,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:46,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501025992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:46,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:46,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:46,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501025992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:46,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:46,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:46,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:46,110 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/a4cd53c0c71c4eee826ca251483290b9 2024-12-06T16:02:46,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/84f341b86b6c46bf95096a34ef48f390 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/84f341b86b6c46bf95096a34ef48f390 2024-12-06T16:02:46,127 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/84f341b86b6c46bf95096a34ef48f390, entries=150, sequenceid=246, filesize=11.9 K 2024-12-06T16:02:46,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/18fe10b910e3426b855f7b0d7a4542a9 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/18fe10b910e3426b855f7b0d7a4542a9 2024-12-06T16:02:46,131 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/18fe10b910e3426b855f7b0d7a4542a9, entries=150, sequenceid=246, filesize=11.9 K 2024-12-06T16:02:46,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/a4cd53c0c71c4eee826ca251483290b9 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/a4cd53c0c71c4eee826ca251483290b9 2024-12-06T16:02:46,136 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/a4cd53c0c71c4eee826ca251483290b9, entries=150, sequenceid=246, filesize=11.9 K 2024-12-06T16:02:46,137 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for d252d31b6d43033dcd4545dc84403a2e in 551ms, sequenceid=246, compaction requested=false 2024-12-06T16:02:46,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:46,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:46,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-06T16:02:46,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-06T16:02:46,142 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-06T16:02:46,142 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4750 sec 2024-12-06T16:02:46,146 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.4810 sec 2024-12-06T16:02:46,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:46,217 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:02:46,217 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d252d31b6d43033dcd4545dc84403a2e 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-06T16:02:46,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/165f091ea1de4722919a93baffdb1465 is 50, key is test_row_0/A:col10/1733500965869/Put/seqid=0 2024-12-06T16:02:46,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742349_1525 (size=14691) 2024-12-06T16:02:46,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/165f091ea1de4722919a93baffdb1465 2024-12-06T16:02:46,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/be7760d3912b480b8d37e57a1151b390 is 50, key is test_row_0/B:col10/1733500965869/Put/seqid=0 2024-12-06T16:02:46,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742350_1526 (size=12251) 2024-12-06T16:02:46,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:46,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501026303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:46,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:46,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501026318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:46,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:46,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501026319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:46,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:46,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501026320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:46,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:46,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501026320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:46,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:46,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:46,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501026431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:46,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:46,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501026431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:46,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:46,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:46,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501026442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:46,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:46,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501026442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:46,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:46,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:46,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501026442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:46,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:46,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:46,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/be7760d3912b480b8d37e57a1151b390 2024-12-06T16:02:46,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:46,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501026642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:46,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/fc95d6a29bb648f5a214d6e1164c3658 is 50, key is test_row_0/C:col10/1733500965869/Put/seqid=0 2024-12-06T16:02:46,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:46,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742351_1527 (size=12251) 2024-12-06T16:02:46,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:46,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501026652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:46,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:46,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:46,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501026662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:46,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:46,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:46,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501026662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:46,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:46,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:46,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501026662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:46,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:46,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-06T16:02:46,812 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:02:46,813 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:02:46,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-06T16:02:46,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-06T16:02:46,815 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:02:46,815 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:02:46,816 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:02:46,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-06T16:02:46,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-06T16:02:46,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:46,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:46,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:46,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:46,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:46,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:46,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:46,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501026962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:46,973 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:46,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:46,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501026972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:46,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:46,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:46,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501026981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:46,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:46,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:46,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501026981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:46,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:46,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:46,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501026982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:46,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:46,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:47,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/fc95d6a29bb648f5a214d6e1164c3658 2024-12-06T16:02:47,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/165f091ea1de4722919a93baffdb1465 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/165f091ea1de4722919a93baffdb1465 2024-12-06T16:02:47,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/165f091ea1de4722919a93baffdb1465, entries=200, sequenceid=261, filesize=14.3 K 2024-12-06T16:02:47,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/be7760d3912b480b8d37e57a1151b390 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/be7760d3912b480b8d37e57a1151b390 2024-12-06T16:02:47,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/be7760d3912b480b8d37e57a1151b390, entries=150, sequenceid=261, filesize=12.0 K 2024-12-06T16:02:47,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/fc95d6a29bb648f5a214d6e1164c3658 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/fc95d6a29bb648f5a214d6e1164c3658 2024-12-06T16:02:47,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/fc95d6a29bb648f5a214d6e1164c3658, entries=150, sequenceid=261, filesize=12.0 K 2024-12-06T16:02:47,070 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for d252d31b6d43033dcd4545dc84403a2e in 853ms, sequenceid=261, compaction requested=true 2024-12-06T16:02:47,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:47,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d252d31b6d43033dcd4545dc84403a2e:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:02:47,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:47,070 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:47,070 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:47,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d252d31b6d43033dcd4545dc84403a2e:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:02:47,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:47,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d252d31b6d43033dcd4545dc84403a2e:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:02:47,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:47,071 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39539 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:47,071 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): d252d31b6d43033dcd4545dc84403a2e/A is initiating minor compaction (all files) 2024-12-06T16:02:47,071 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d252d31b6d43033dcd4545dc84403a2e/A in TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:47,071 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/94c7898da2a94f12af8a21b48dcc20f2, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/84f341b86b6c46bf95096a34ef48f390, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/165f091ea1de4722919a93baffdb1465] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp, totalSize=38.6 K 2024-12-06T16:02:47,071 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:47,071 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): d252d31b6d43033dcd4545dc84403a2e/B is initiating minor compaction (all files) 2024-12-06T16:02:47,071 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d252d31b6d43033dcd4545dc84403a2e/B in TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:47,071 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/b034e79856534fbeb7b7f866dc7f776d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/18fe10b910e3426b855f7b0d7a4542a9, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/be7760d3912b480b8d37e57a1151b390] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp, totalSize=36.2 K 2024-12-06T16:02:47,072 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94c7898da2a94f12af8a21b48dcc20f2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733500964610 2024-12-06T16:02:47,072 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting b034e79856534fbeb7b7f866dc7f776d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733500964610 2024-12-06T16:02:47,072 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84f341b86b6c46bf95096a34ef48f390, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733500964681 2024-12-06T16:02:47,072 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 18fe10b910e3426b855f7b0d7a4542a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733500964681 2024-12-06T16:02:47,072 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting be7760d3912b480b8d37e57a1151b390, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733500965869 2024-12-06T16:02:47,072 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 165f091ea1de4722919a93baffdb1465, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733500965869 2024-12-06T16:02:47,090 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d252d31b6d43033dcd4545dc84403a2e#B#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:47,090 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/b56dfeda4f8940c78aefc0c044a2c89c is 50, key is test_row_0/B:col10/1733500965869/Put/seqid=0 2024-12-06T16:02:47,092 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d252d31b6d43033dcd4545dc84403a2e#A#compaction#451 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:47,093 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/d459cb5ebb4e4d8586cca01d71edd37e is 50, key is test_row_0/A:col10/1733500965869/Put/seqid=0 2024-12-06T16:02:47,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742352_1528 (size=12899) 2024-12-06T16:02:47,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742353_1529 (size=12899) 2024-12-06T16:02:47,105 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/b56dfeda4f8940c78aefc0c044a2c89c as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/b56dfeda4f8940c78aefc0c044a2c89c 2024-12-06T16:02:47,105 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/d459cb5ebb4e4d8586cca01d71edd37e as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/d459cb5ebb4e4d8586cca01d71edd37e 2024-12-06T16:02:47,110 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d252d31b6d43033dcd4545dc84403a2e/A of d252d31b6d43033dcd4545dc84403a2e into d459cb5ebb4e4d8586cca01d71edd37e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:47,110 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d252d31b6d43033dcd4545dc84403a2e/B of d252d31b6d43033dcd4545dc84403a2e into b56dfeda4f8940c78aefc0c044a2c89c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:47,110 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:47,110 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:47,110 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., storeName=d252d31b6d43033dcd4545dc84403a2e/A, priority=13, startTime=1733500967070; duration=0sec 2024-12-06T16:02:47,110 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., storeName=d252d31b6d43033dcd4545dc84403a2e/B, priority=13, startTime=1733500967070; duration=0sec 2024-12-06T16:02:47,110 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:47,110 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d252d31b6d43033dcd4545dc84403a2e:A 2024-12-06T16:02:47,110 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:47,110 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:47,110 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d252d31b6d43033dcd4545dc84403a2e:B 2024-12-06T16:02:47,112 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:47,112 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): d252d31b6d43033dcd4545dc84403a2e/C is initiating minor compaction (all files) 2024-12-06T16:02:47,112 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d252d31b6d43033dcd4545dc84403a2e/C in TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:47,112 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/89cdf2a1301441b9b10f23462fd1ba03, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/a4cd53c0c71c4eee826ca251483290b9, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/fc95d6a29bb648f5a214d6e1164c3658] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp, totalSize=36.2 K 2024-12-06T16:02:47,112 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89cdf2a1301441b9b10f23462fd1ba03, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733500964610 2024-12-06T16:02:47,113 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4cd53c0c71c4eee826ca251483290b9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733500964681 2024-12-06T16:02:47,113 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc95d6a29bb648f5a214d6e1164c3658, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733500965869 2024-12-06T16:02:47,119 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d252d31b6d43033dcd4545dc84403a2e#C#compaction#452 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:47,119 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/ba8fc8d6f5d14431989a8033bb883ec2 is 50, key is test_row_0/C:col10/1733500965869/Put/seqid=0 2024-12-06T16:02:47,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742354_1530 (size=12899) 2024-12-06T16:02:47,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-06T16:02:47,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:47,122 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing d252d31b6d43033dcd4545dc84403a2e 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-06T16:02:47,130 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/ba8fc8d6f5d14431989a8033bb883ec2 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/ba8fc8d6f5d14431989a8033bb883ec2 2024-12-06T16:02:47,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-06T16:02:47,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/f975113cb1ca4df3b36d55d7c3c6bfb6 is 50, key is test_row_0/A:col10/1733500966318/Put/seqid=0 2024-12-06T16:02:47,136 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d252d31b6d43033dcd4545dc84403a2e/C of d252d31b6d43033dcd4545dc84403a2e into ba8fc8d6f5d14431989a8033bb883ec2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:47,136 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:47,136 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., storeName=d252d31b6d43033dcd4545dc84403a2e/C, priority=13, startTime=1733500967070; duration=0sec 2024-12-06T16:02:47,136 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:47,136 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d252d31b6d43033dcd4545dc84403a2e:C 2024-12-06T16:02:47,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742355_1531 (size=12301) 2024-12-06T16:02:47,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-06T16:02:47,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:47,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:47,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501027516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:47,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:47,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:47,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:47,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501027517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:47,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501027517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:47,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501027518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:47,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:47,533 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:47,533 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:47,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501027520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:47,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:47,558 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/f975113cb1ca4df3b36d55d7c3c6bfb6 2024-12-06T16:02:47,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/83bfa520cf26417eaf0fc34e57cd2abe is 50, key is test_row_0/B:col10/1733500966318/Put/seqid=0 2024-12-06T16:02:47,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742356_1532 (size=12301) 2024-12-06T16:02:47,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501027621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:47,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:47,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:47,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:47,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501027632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:47,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:47,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:47,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:47,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501027642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:47,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:47,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:47,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:47,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501027642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:47,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501027643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:47,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:47,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501027832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:47,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:47,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:47,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:47,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:47,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501027851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:47,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:47,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:47,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:47,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:47,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501027862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:47,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:47,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501027862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:47,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:47,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:47,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:47,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:47,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501027862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:47,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:47,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:47,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:47,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:47,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:47,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:47,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-06T16:02:47,987 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/83bfa520cf26417eaf0fc34e57cd2abe 2024-12-06T16:02:47,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/a9cefb93c23e4933915cceab606a07c9 is 50, key is test_row_0/C:col10/1733500966318/Put/seqid=0 2024-12-06T16:02:48,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742357_1533 (size=12301) 2024-12-06T16:02:48,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:48,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501028152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:48,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:48,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:48,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:48,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:48,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501028162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:48,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:48,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:48,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:48,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:48,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501028182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:48,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:48,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:48,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:48,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:48,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501028182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:48,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:48,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:48,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:48,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:48,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501028182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:48,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:48,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:48,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:48,408 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/a9cefb93c23e4933915cceab606a07c9 2024-12-06T16:02:48,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/f975113cb1ca4df3b36d55d7c3c6bfb6 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/f975113cb1ca4df3b36d55d7c3c6bfb6 2024-12-06T16:02:48,418 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/f975113cb1ca4df3b36d55d7c3c6bfb6, entries=150, sequenceid=286, filesize=12.0 K 2024-12-06T16:02:48,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/83bfa520cf26417eaf0fc34e57cd2abe as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/83bfa520cf26417eaf0fc34e57cd2abe 2024-12-06T16:02:48,423 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/83bfa520cf26417eaf0fc34e57cd2abe, entries=150, sequenceid=286, filesize=12.0 K 2024-12-06T16:02:48,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/a9cefb93c23e4933915cceab606a07c9 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/a9cefb93c23e4933915cceab606a07c9 2024-12-06T16:02:48,427 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/a9cefb93c23e4933915cceab606a07c9, entries=150, sequenceid=286, filesize=12.0 K 2024-12-06T16:02:48,428 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for d252d31b6d43033dcd4545dc84403a2e in 1306ms, sequenceid=286, compaction requested=false 2024-12-06T16:02:48,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:48,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:48,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-06T16:02:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-06T16:02:48,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-06T16:02:48,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6140 sec 2024-12-06T16:02:48,434 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.6200 sec 2024-12-06T16:02:48,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:48,680 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:02:48,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d252d31b6d43033dcd4545dc84403a2e 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-06T16:02:48,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/f94448ecfc3e4743856e860ec677fb45 is 50, key is test_row_0/A:col10/1733500967515/Put/seqid=0 2024-12-06T16:02:48,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742358_1534 (size=12301) 2024-12-06T16:02:48,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/f94448ecfc3e4743856e860ec677fb45 2024-12-06T16:02:48,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/f65d8221a4974694986d99017f7cc2df is 50, key is test_row_0/B:col10/1733500967515/Put/seqid=0 2024-12-06T16:02:48,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742359_1535 (size=12301) 2024-12-06T16:02:48,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/f65d8221a4974694986d99017f7cc2df 2024-12-06T16:02:48,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/0996cf3d684247ebb380d76541da0456 is 50, key is test_row_0/C:col10/1733500967515/Put/seqid=0 2024-12-06T16:02:48,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742360_1536 (size=12301) 2024-12-06T16:02:48,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:48,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501028731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:48,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:48,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:48,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:48,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:48,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501028735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:48,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:48,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501028737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:48,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:48,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:48,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:48,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:48,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501028738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:48,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:48,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501028741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:48,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:48,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:48,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:48,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:48,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:48,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:48,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:48,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501028852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:48,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:48,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:48,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:48,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:48,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501028862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:48,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:48,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:48,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501028862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:48,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:48,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:48,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:48,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:48,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501028862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:48,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:48,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:48,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:48,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:48,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:48,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:48,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501028862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:48,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:48,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:48,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:48,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-06T16:02:48,962 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:02:48,963 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:02:48,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-06T16:02:48,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-06T16:02:48,965 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:02:48,965 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:02:48,965 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:02:49,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-06T16:02:49,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:49,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501029072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:49,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:49,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:49,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501029082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:49,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:49,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:49,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501029082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:49,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:49,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501029082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:49,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:49,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501029082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:49,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:49,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:49,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:49,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-06T16:02:49,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:49,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:49,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:49,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:49,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:49,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:49,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/0996cf3d684247ebb380d76541da0456 2024-12-06T16:02:49,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/f94448ecfc3e4743856e860ec677fb45 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/f94448ecfc3e4743856e860ec677fb45 2024-12-06T16:02:49,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/f94448ecfc3e4743856e860ec677fb45, entries=150, sequenceid=301, filesize=12.0 K 2024-12-06T16:02:49,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/f65d8221a4974694986d99017f7cc2df as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/f65d8221a4974694986d99017f7cc2df 2024-12-06T16:02:49,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/f65d8221a4974694986d99017f7cc2df, entries=150, sequenceid=301, filesize=12.0 K 2024-12-06T16:02:49,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/0996cf3d684247ebb380d76541da0456 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/0996cf3d684247ebb380d76541da0456 2024-12-06T16:02:49,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/0996cf3d684247ebb380d76541da0456, entries=150, sequenceid=301, filesize=12.0 K 2024-12-06T16:02:49,159 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for d252d31b6d43033dcd4545dc84403a2e in 479ms, sequenceid=301, compaction requested=true 2024-12-06T16:02:49,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:49,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d252d31b6d43033dcd4545dc84403a2e:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:02:49,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:49,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d252d31b6d43033dcd4545dc84403a2e:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:02:49,160 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:49,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:49,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d252d31b6d43033dcd4545dc84403a2e:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:02:49,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-06T16:02:49,160 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:49,160 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:49,161 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): d252d31b6d43033dcd4545dc84403a2e/A is initiating minor compaction (all files) 2024-12-06T16:02:49,161 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d252d31b6d43033dcd4545dc84403a2e/A in TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:49,161 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/d459cb5ebb4e4d8586cca01d71edd37e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/f975113cb1ca4df3b36d55d7c3c6bfb6, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/f94448ecfc3e4743856e860ec677fb45] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp, totalSize=36.6 K 2024-12-06T16:02:49,161 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting d459cb5ebb4e4d8586cca01d71edd37e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733500965869 2024-12-06T16:02:49,161 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:49,161 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): d252d31b6d43033dcd4545dc84403a2e/B is initiating minor compaction (all files) 2024-12-06T16:02:49,161 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d252d31b6d43033dcd4545dc84403a2e/B in TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:49,162 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/b56dfeda4f8940c78aefc0c044a2c89c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/83bfa520cf26417eaf0fc34e57cd2abe, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/f65d8221a4974694986d99017f7cc2df] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp, totalSize=36.6 K 2024-12-06T16:02:49,162 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting f975113cb1ca4df3b36d55d7c3c6bfb6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733500966301 2024-12-06T16:02:49,162 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting b56dfeda4f8940c78aefc0c044a2c89c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733500965869 2024-12-06T16:02:49,163 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting f94448ecfc3e4743856e860ec677fb45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733500967515 2024-12-06T16:02:49,163 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83bfa520cf26417eaf0fc34e57cd2abe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733500966301 2024-12-06T16:02:49,164 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting f65d8221a4974694986d99017f7cc2df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733500967515 2024-12-06T16:02:49,170 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d252d31b6d43033dcd4545dc84403a2e#A#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:49,171 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/78e5fd5b9c5945c68f9f8d35b55b3cd7 is 50, key is test_row_0/A:col10/1733500967515/Put/seqid=0 2024-12-06T16:02:49,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742361_1537 (size=13051) 2024-12-06T16:02:49,180 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d252d31b6d43033dcd4545dc84403a2e#B#compaction#460 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:49,181 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/6b1d7d1f799e478a9745331db61a3055 is 50, key is test_row_0/B:col10/1733500967515/Put/seqid=0 2024-12-06T16:02:49,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742362_1538 (size=13051) 2024-12-06T16:02:49,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-06T16:02:49,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:49,272 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing d252d31b6d43033dcd4545dc84403a2e 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-06T16:02:49,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/f4c956ce052f438ba779994db7c018f3 is 50, key is test_row_0/A:col10/1733500968738/Put/seqid=0 2024-12-06T16:02:49,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-06T16:02:49,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742363_1539 (size=12301) 2024-12-06T16:02:49,296 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/f4c956ce052f438ba779994db7c018f3 2024-12-06T16:02:49,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/dd8c75f84c1c4656b3e165d94ada4834 is 50, key is test_row_0/B:col10/1733500968738/Put/seqid=0 2024-12-06T16:02:49,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742364_1540 (size=12301) 2024-12-06T16:02:49,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:49,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:49,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:49,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501029415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:49,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501029416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:49,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:49,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501029425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:49,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:49,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501029427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:49,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:49,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:49,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501029428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:49,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:49,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:49,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501029532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:49,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:49,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501029542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:49,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:49,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501029542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:49,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:49,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:49,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501029542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:49,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:49,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:49,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:49,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501029552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:49,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:49,581 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/78e5fd5b9c5945c68f9f8d35b55b3cd7 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/78e5fd5b9c5945c68f9f8d35b55b3cd7 2024-12-06T16:02:49,586 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d252d31b6d43033dcd4545dc84403a2e/A of d252d31b6d43033dcd4545dc84403a2e into 78e5fd5b9c5945c68f9f8d35b55b3cd7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:49,586 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:49,586 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., storeName=d252d31b6d43033dcd4545dc84403a2e/A, priority=13, startTime=1733500969159; duration=0sec 2024-12-06T16:02:49,586 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:49,586 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d252d31b6d43033dcd4545dc84403a2e:A 2024-12-06T16:02:49,586 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:49,588 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:49,588 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): d252d31b6d43033dcd4545dc84403a2e/C is initiating minor compaction (all files) 2024-12-06T16:02:49,588 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d252d31b6d43033dcd4545dc84403a2e/C in TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:49,588 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/ba8fc8d6f5d14431989a8033bb883ec2, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/a9cefb93c23e4933915cceab606a07c9, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/0996cf3d684247ebb380d76541da0456] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp, totalSize=36.6 K 2024-12-06T16:02:49,589 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting ba8fc8d6f5d14431989a8033bb883ec2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733500965869 2024-12-06T16:02:49,589 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting a9cefb93c23e4933915cceab606a07c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733500966301 2024-12-06T16:02:49,589 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 0996cf3d684247ebb380d76541da0456, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733500967515 2024-12-06T16:02:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-06T16:02:49,597 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d252d31b6d43033dcd4545dc84403a2e#C#compaction#463 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:49,598 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/c82d0d4899eb4b0fb512a838ab06e2e0 is 50, key is test_row_0/C:col10/1733500967515/Put/seqid=0 2024-12-06T16:02:49,600 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/6b1d7d1f799e478a9745331db61a3055 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/6b1d7d1f799e478a9745331db61a3055 2024-12-06T16:02:49,604 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d252d31b6d43033dcd4545dc84403a2e/B of d252d31b6d43033dcd4545dc84403a2e into 6b1d7d1f799e478a9745331db61a3055(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:49,604 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:49,604 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., storeName=d252d31b6d43033dcd4545dc84403a2e/B, priority=13, startTime=1733500969160; duration=0sec 2024-12-06T16:02:49,604 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:49,604 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d252d31b6d43033dcd4545dc84403a2e:B 2024-12-06T16:02:49,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742365_1541 (size=13051) 2024-12-06T16:02:49,710 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/dd8c75f84c1c4656b3e165d94ada4834 2024-12-06T16:02:49,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/1ef8a29293284370bbf9aea91fbce07c is 50, key is test_row_0/C:col10/1733500968738/Put/seqid=0 2024-12-06T16:02:49,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742366_1542 (size=12301) 2024-12-06T16:02:49,731 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/1ef8a29293284370bbf9aea91fbce07c 2024-12-06T16:02:49,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/f4c956ce052f438ba779994db7c018f3 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/f4c956ce052f438ba779994db7c018f3 2024-12-06T16:02:49,746 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/f4c956ce052f438ba779994db7c018f3, entries=150, sequenceid=323, filesize=12.0 K 2024-12-06T16:02:49,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/dd8c75f84c1c4656b3e165d94ada4834 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/dd8c75f84c1c4656b3e165d94ada4834 2024-12-06T16:02:49,751 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/dd8c75f84c1c4656b3e165d94ada4834, entries=150, sequenceid=323, filesize=12.0 K 2024-12-06T16:02:49,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/1ef8a29293284370bbf9aea91fbce07c as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/1ef8a29293284370bbf9aea91fbce07c 2024-12-06T16:02:49,756 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/1ef8a29293284370bbf9aea91fbce07c, entries=150, sequenceid=323, filesize=12.0 K 2024-12-06T16:02:49,757 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for d252d31b6d43033dcd4545dc84403a2e in 485ms, sequenceid=323, compaction requested=false 2024-12-06T16:02:49,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:49,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:49,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-06T16:02:49,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-06T16:02:49,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-06T16:02:49,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 794 msec 2024-12-06T16:02:49,762 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 798 msec 2024-12-06T16:02:49,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:49,763 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:02:49,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d252d31b6d43033dcd4545dc84403a2e 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-06T16:02:49,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/6cf937ef57714bc79e1a32aaa496eadf is 50, key is test_row_0/A:col10/1733500969425/Put/seqid=0 2024-12-06T16:02:49,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742367_1543 (size=14741) 2024-12-06T16:02:49,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/6cf937ef57714bc79e1a32aaa496eadf 2024-12-06T16:02:49,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/7800e153e6744e929135da77d9b1566a is 50, key is test_row_0/B:col10/1733500969425/Put/seqid=0 2024-12-06T16:02:49,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742368_1544 (size=12301) 2024-12-06T16:02:49,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:49,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501029801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:49,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:49,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501029809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:49,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:49,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:49,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:49,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501029817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:49,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:49,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:49,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501029818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:49,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:49,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501029819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:49,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:49,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:49,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:49,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501029922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:49,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:49,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501029922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:49,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:49,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:49,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501029931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:49,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:49,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:49,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501029942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:49,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:49,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501029942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:49,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:49,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:49,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:50,014 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/c82d0d4899eb4b0fb512a838ab06e2e0 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/c82d0d4899eb4b0fb512a838ab06e2e0 2024-12-06T16:02:50,021 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d252d31b6d43033dcd4545dc84403a2e/C of d252d31b6d43033dcd4545dc84403a2e into c82d0d4899eb4b0fb512a838ab06e2e0(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:50,021 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:50,021 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., storeName=d252d31b6d43033dcd4545dc84403a2e/C, priority=13, startTime=1733500969160; duration=0sec 2024-12-06T16:02:50,022 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:50,022 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d252d31b6d43033dcd4545dc84403a2e:C 2024-12-06T16:02:50,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-06T16:02:50,102 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:02:50,103 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:02:50,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-06T16:02:50,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-06T16:02:50,105 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:02:50,105 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:02:50,106 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:02:50,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:50,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501030142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:50,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:50,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501030142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501030142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:50,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:50,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501030161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:50,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:50,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501030161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:50,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:50,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:50,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/7800e153e6744e929135da77d9b1566a 2024-12-06T16:02:50,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/7df9251e801e4233860c60e90fe8e7e4 is 50, key is test_row_0/C:col10/1733500969425/Put/seqid=0 2024-12-06T16:02:50,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742369_1545 (size=12301) 2024-12-06T16:02:50,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/7df9251e801e4233860c60e90fe8e7e4 2024-12-06T16:02:50,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/6cf937ef57714bc79e1a32aaa496eadf as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/6cf937ef57714bc79e1a32aaa496eadf 2024-12-06T16:02:50,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/6cf937ef57714bc79e1a32aaa496eadf, entries=200, sequenceid=340, filesize=14.4 K 2024-12-06T16:02:50,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/7800e153e6744e929135da77d9b1566a as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/7800e153e6744e929135da77d9b1566a 2024-12-06T16:02:50,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-06T16:02:50,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/7800e153e6744e929135da77d9b1566a, entries=150, sequenceid=340, filesize=12.0 K 2024-12-06T16:02:50,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/7df9251e801e4233860c60e90fe8e7e4 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/7df9251e801e4233860c60e90fe8e7e4 2024-12-06T16:02:50,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/7df9251e801e4233860c60e90fe8e7e4, entries=150, sequenceid=340, filesize=12.0 K 2024-12-06T16:02:50,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for d252d31b6d43033dcd4545dc84403a2e in 457ms, sequenceid=340, compaction requested=true 2024-12-06T16:02:50,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:50,221 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:50,222 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:50,222 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): d252d31b6d43033dcd4545dc84403a2e/A is initiating minor compaction (all files) 2024-12-06T16:02:50,222 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d252d31b6d43033dcd4545dc84403a2e/A in TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:50,222 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/78e5fd5b9c5945c68f9f8d35b55b3cd7, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/f4c956ce052f438ba779994db7c018f3, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/6cf937ef57714bc79e1a32aaa496eadf] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp, totalSize=39.2 K 2024-12-06T16:02:50,222 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78e5fd5b9c5945c68f9f8d35b55b3cd7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733500967515 2024-12-06T16:02:50,222 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4c956ce052f438ba779994db7c018f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733500968718 2024-12-06T16:02:50,223 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cf937ef57714bc79e1a32aaa496eadf, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733500969412 2024-12-06T16:02:50,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d252d31b6d43033dcd4545dc84403a2e:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:02:50,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:50,227 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:50,229 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d252d31b6d43033dcd4545dc84403a2e#A#compaction#468 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:50,230 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/e1ae0a1ade5f49eea01324833130174a is 50, key is test_row_0/A:col10/1733500969425/Put/seqid=0 2024-12-06T16:02:50,231 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:50,231 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): d252d31b6d43033dcd4545dc84403a2e/B is initiating minor compaction (all files) 2024-12-06T16:02:50,231 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d252d31b6d43033dcd4545dc84403a2e/B in TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:50,231 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/6b1d7d1f799e478a9745331db61a3055, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/dd8c75f84c1c4656b3e165d94ada4834, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/7800e153e6744e929135da77d9b1566a] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp, totalSize=36.8 K 2024-12-06T16:02:50,231 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b1d7d1f799e478a9745331db61a3055, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733500967515 2024-12-06T16:02:50,232 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting dd8c75f84c1c4656b3e165d94ada4834, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733500968718 2024-12-06T16:02:50,232 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 7800e153e6744e929135da77d9b1566a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733500969414 2024-12-06T16:02:50,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d252d31b6d43033dcd4545dc84403a2e:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:02:50,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:50,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d252d31b6d43033dcd4545dc84403a2e:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:02:50,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:50,239 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d252d31b6d43033dcd4545dc84403a2e#B#compaction#469 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:50,239 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/1f09e6efa6ea4eb6ba03e1ee4225ffa7 is 50, key is test_row_0/B:col10/1733500969425/Put/seqid=0 2024-12-06T16:02:50,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742370_1546 (size=13153) 2024-12-06T16:02:50,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-06T16:02:50,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:50,258 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing d252d31b6d43033dcd4545dc84403a2e 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-06T16:02:50,260 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/e1ae0a1ade5f49eea01324833130174a as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/e1ae0a1ade5f49eea01324833130174a 2024-12-06T16:02:50,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742371_1547 (size=13153) 2024-12-06T16:02:50,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/7762f47b3578403da0733da5efcab3d0 is 50, key is test_row_0/A:col10/1733500969799/Put/seqid=0 2024-12-06T16:02:50,274 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/1f09e6efa6ea4eb6ba03e1ee4225ffa7 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/1f09e6efa6ea4eb6ba03e1ee4225ffa7 2024-12-06T16:02:50,274 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d252d31b6d43033dcd4545dc84403a2e/A of d252d31b6d43033dcd4545dc84403a2e into e1ae0a1ade5f49eea01324833130174a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:50,274 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:50,274 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., storeName=d252d31b6d43033dcd4545dc84403a2e/A, priority=13, startTime=1733500970221; duration=0sec 2024-12-06T16:02:50,274 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:50,274 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d252d31b6d43033dcd4545dc84403a2e:A 2024-12-06T16:02:50,274 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:50,277 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:50,277 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): d252d31b6d43033dcd4545dc84403a2e/C is initiating minor compaction (all files) 2024-12-06T16:02:50,277 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d252d31b6d43033dcd4545dc84403a2e/C in TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:50,277 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/c82d0d4899eb4b0fb512a838ab06e2e0, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/1ef8a29293284370bbf9aea91fbce07c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/7df9251e801e4233860c60e90fe8e7e4] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp, totalSize=36.8 K 2024-12-06T16:02:50,278 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting c82d0d4899eb4b0fb512a838ab06e2e0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733500967515 2024-12-06T16:02:50,279 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ef8a29293284370bbf9aea91fbce07c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733500968718 2024-12-06T16:02:50,279 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7df9251e801e4233860c60e90fe8e7e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733500969414 2024-12-06T16:02:50,283 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d252d31b6d43033dcd4545dc84403a2e/B of d252d31b6d43033dcd4545dc84403a2e into 1f09e6efa6ea4eb6ba03e1ee4225ffa7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:50,283 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:50,283 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., storeName=d252d31b6d43033dcd4545dc84403a2e/B, priority=13, startTime=1733500970227; duration=0sec 2024-12-06T16:02:50,283 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:50,283 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d252d31b6d43033dcd4545dc84403a2e:B 2024-12-06T16:02:50,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742372_1548 (size=12301) 2024-12-06T16:02:50,292 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d252d31b6d43033dcd4545dc84403a2e#C#compaction#471 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:50,293 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/27f091a29b214277bd74e59abf9ad80b is 50, key is test_row_0/C:col10/1733500969425/Put/seqid=0 2024-12-06T16:02:50,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742373_1549 (size=13153) 2024-12-06T16:02:50,315 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/27f091a29b214277bd74e59abf9ad80b as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/27f091a29b214277bd74e59abf9ad80b 2024-12-06T16:02:50,321 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d252d31b6d43033dcd4545dc84403a2e/C of d252d31b6d43033dcd4545dc84403a2e into 27f091a29b214277bd74e59abf9ad80b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:50,321 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:50,321 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., storeName=d252d31b6d43033dcd4545dc84403a2e/C, priority=13, startTime=1733500970237; duration=0sec 2024-12-06T16:02:50,322 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:50,322 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d252d31b6d43033dcd4545dc84403a2e:C 2024-12-06T16:02:50,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-06T16:02:50,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:50,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:50,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:50,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501030483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:50,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:50,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501030490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:50,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:50,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:50,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:50,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501030491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:50,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:50,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501030495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:50,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:50,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501030496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:50,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:50,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:50,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501030601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:50,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:50,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501030601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:50,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:50,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:50,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:50,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501030612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:50,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:50,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501030612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:50,621 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,621 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,621 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:50,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:50,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501030612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:50,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:50,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:50,688 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/7762f47b3578403da0733da5efcab3d0 2024-12-06T16:02:50,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/1d7854ca0ec749b5838153d9e3ad1adb is 50, key is test_row_0/B:col10/1733500969799/Put/seqid=0 2024-12-06T16:02:50,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742374_1550 (size=12301) 2024-12-06T16:02:50,701 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/1d7854ca0ec749b5838153d9e3ad1adb 2024-12-06T16:02:50,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/e217722422d74343954d675c03a4b89c is 50, key is test_row_0/C:col10/1733500969799/Put/seqid=0 2024-12-06T16:02:50,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742375_1551 (size=12301) 2024-12-06T16:02:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-06T16:02:50,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501030812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:50,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501030812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:50,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:50,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:50,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:50,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501030831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:50,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:50,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501030832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:50,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:50,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:50,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:50,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501030832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:50,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:50,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:51,114 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/e217722422d74343954d675c03a4b89c 2024-12-06T16:02:51,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/7762f47b3578403da0733da5efcab3d0 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/7762f47b3578403da0733da5efcab3d0 2024-12-06T16:02:51,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:51,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501031122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:51,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:51,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:51,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:51,134 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/7762f47b3578403da0733da5efcab3d0, entries=150, sequenceid=362, filesize=12.0 K 2024-12-06T16:02:51,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:51,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501031124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:51,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:51,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:51,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:51,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/1d7854ca0ec749b5838153d9e3ad1adb as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/1d7854ca0ec749b5838153d9e3ad1adb 2024-12-06T16:02:51,141 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/1d7854ca0ec749b5838153d9e3ad1adb, entries=150, sequenceid=362, filesize=12.0 K 2024-12-06T16:02:51,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/e217722422d74343954d675c03a4b89c as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/e217722422d74343954d675c03a4b89c 2024-12-06T16:02:51,146 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/e217722422d74343954d675c03a4b89c, entries=150, sequenceid=362, filesize=12.0 K 2024-12-06T16:02:51,147 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for d252d31b6d43033dcd4545dc84403a2e in 889ms, sequenceid=362, compaction requested=false 2024-12-06T16:02:51,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:51,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:51,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-06T16:02:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-06T16:02:51,150 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-06T16:02:51,150 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0430 sec 2024-12-06T16:02:51,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.0470 sec 2024-12-06T16:02:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:51,157 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:02:51,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d252d31b6d43033dcd4545dc84403a2e 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-12-06T16:02:51,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/19946b7fd249488fa20a44a25915ffd6 is 50, key is test_row_0/A:col10/1733500971156/Put/seqid=0 2024-12-06T16:02:51,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742376_1552 (size=14741) 2024-12-06T16:02:51,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:51,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501031219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:51,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:51,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:51,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:51,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-06T16:02:51,242 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:02:51,242 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:02:51,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-06T16:02:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-06T16:02:51,244 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:02:51,245 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:02:51,245 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:02:51,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501031234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:51,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:51,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:51,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501031235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:51,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:51,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:51,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:51,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-06T16:02:51,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501031342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:51,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:51,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:51,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:51,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501031361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:51,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501031361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:51,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:51,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:51,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:51,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:51,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:51,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:51,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-06T16:02:51,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:51,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:51,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:51,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:51,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:51,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-06T16:02:51,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:51,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:51,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:51,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:51,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-06T16:02:51,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501031563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:51,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:51,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:51,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:51,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/19946b7fd249488fa20a44a25915ffd6 2024-12-06T16:02:51,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/4187074220094f46a65e684ebbb45b46 is 50, key is test_row_0/B:col10/1733500971156/Put/seqid=0 2024-12-06T16:02:51,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742377_1553 (size=12301) 2024-12-06T16:02:51,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501031581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:51,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:51,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:51,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:51,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501031582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:51,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:51,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:51,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:51,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501031642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:51,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501031642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:51,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:51,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:51,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:51,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:51,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:51,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:51,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-06T16:02:51,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:51,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:51,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:51,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:51,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:51,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-06T16:02:51,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:51,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:51,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:51,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:51,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-06T16:02:51,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501031881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:51,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:51,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:51,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:51,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:51,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501031902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:51,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501031902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:51,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/4187074220094f46a65e684ebbb45b46 2024-12-06T16:02:51,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/b72e70a06a2e4b44a790560304da054d is 50, key is test_row_0/C:col10/1733500971156/Put/seqid=0 2024-12-06T16:02:52,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742378_1554 (size=12301) 2024-12-06T16:02:52,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-06T16:02:52,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:52,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:52,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:52,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:52,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:52,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:52,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-06T16:02:52,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:52,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:52,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:52,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:52,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:52,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-06T16:02:52,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:52,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:52,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:52,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:52,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-06T16:02:52,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/b72e70a06a2e4b44a790560304da054d 2024-12-06T16:02:52,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/19946b7fd249488fa20a44a25915ffd6 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/19946b7fd249488fa20a44a25915ffd6 2024-12-06T16:02:52,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501032402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:52,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:52,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:52,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:52,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/19946b7fd249488fa20a44a25915ffd6, entries=200, sequenceid=382, filesize=14.4 K 2024-12-06T16:02:52,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/4187074220094f46a65e684ebbb45b46 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/4187074220094f46a65e684ebbb45b46 2024-12-06T16:02:52,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/4187074220094f46a65e684ebbb45b46, entries=150, sequenceid=382, filesize=12.0 K 2024-12-06T16:02:52,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/b72e70a06a2e4b44a790560304da054d as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/b72e70a06a2e4b44a790560304da054d 2024-12-06T16:02:52,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/b72e70a06a2e4b44a790560304da054d, entries=150, sequenceid=382, filesize=12.0 K 2024-12-06T16:02:52,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for d252d31b6d43033dcd4545dc84403a2e in 1261ms, sequenceid=382, compaction requested=true 2024-12-06T16:02:52,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:52,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d252d31b6d43033dcd4545dc84403a2e:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:02:52,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:52,418 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:52,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d252d31b6d43033dcd4545dc84403a2e:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:02:52,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:52,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d252d31b6d43033dcd4545dc84403a2e:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:02:52,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:52,419 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:52,423 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:52,423 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:52,423 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): d252d31b6d43033dcd4545dc84403a2e/A is initiating minor compaction (all files) 2024-12-06T16:02:52,423 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): d252d31b6d43033dcd4545dc84403a2e/B is initiating minor compaction (all files) 2024-12-06T16:02:52,423 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d252d31b6d43033dcd4545dc84403a2e/B in TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:52,423 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d252d31b6d43033dcd4545dc84403a2e/A in TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:52,423 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/1f09e6efa6ea4eb6ba03e1ee4225ffa7, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/1d7854ca0ec749b5838153d9e3ad1adb, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/4187074220094f46a65e684ebbb45b46] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp, totalSize=36.9 K 2024-12-06T16:02:52,423 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/e1ae0a1ade5f49eea01324833130174a, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/7762f47b3578403da0733da5efcab3d0, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/19946b7fd249488fa20a44a25915ffd6] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp, totalSize=39.3 K 2024-12-06T16:02:52,424 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1ae0a1ade5f49eea01324833130174a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733500969414 2024-12-06T16:02:52,424 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f09e6efa6ea4eb6ba03e1ee4225ffa7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733500969414 2024-12-06T16:02:52,424 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d7854ca0ec749b5838153d9e3ad1adb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1733500969799 2024-12-06T16:02:52,424 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7762f47b3578403da0733da5efcab3d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1733500969799 2024-12-06T16:02:52,426 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 4187074220094f46a65e684ebbb45b46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733500970480 2024-12-06T16:02:52,427 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19946b7fd249488fa20a44a25915ffd6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733500970480 2024-12-06T16:02:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:52,427 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:02:52,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d252d31b6d43033dcd4545dc84403a2e 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-12-06T16:02:52,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/56cca3a144274c248161467eae4a88e7 is 50, key is test_row_0/A:col10/1733500972425/Put/seqid=0 2024-12-06T16:02:52,441 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d252d31b6d43033dcd4545dc84403a2e#A#compaction#478 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:52,442 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/abf4d4218ae04975a412859bcba15383 is 50, key is test_row_0/A:col10/1733500971156/Put/seqid=0 2024-12-06T16:02:52,450 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d252d31b6d43033dcd4545dc84403a2e#B#compaction#479 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:52,450 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/b5616538f7284591bfd21f1d1e682ea6 is 50, key is test_row_0/B:col10/1733500971156/Put/seqid=0 2024-12-06T16:02:52,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742379_1555 (size=12301) 2024-12-06T16:02:52,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742380_1556 (size=13255) 2024-12-06T16:02:52,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742381_1557 (size=13255) 2024-12-06T16:02:52,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-06T16:02:52,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:52,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:52,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:52,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:52,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:52,477 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/abf4d4218ae04975a412859bcba15383 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/abf4d4218ae04975a412859bcba15383 2024-12-06T16:02:52,480 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/b5616538f7284591bfd21f1d1e682ea6 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/b5616538f7284591bfd21f1d1e682ea6 2024-12-06T16:02:52,482 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d252d31b6d43033dcd4545dc84403a2e/A of d252d31b6d43033dcd4545dc84403a2e into abf4d4218ae04975a412859bcba15383(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:52,482 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:52,482 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., storeName=d252d31b6d43033dcd4545dc84403a2e/A, priority=13, startTime=1733500972418; duration=0sec 2024-12-06T16:02:52,482 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:52,482 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d252d31b6d43033dcd4545dc84403a2e:A 2024-12-06T16:02:52,482 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:02:52,483 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:02:52,483 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): d252d31b6d43033dcd4545dc84403a2e/C is initiating minor compaction (all files) 2024-12-06T16:02:52,483 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d252d31b6d43033dcd4545dc84403a2e/C in TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:52,483 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/27f091a29b214277bd74e59abf9ad80b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/e217722422d74343954d675c03a4b89c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/b72e70a06a2e4b44a790560304da054d] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp, totalSize=36.9 K 2024-12-06T16:02:52,484 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27f091a29b214277bd74e59abf9ad80b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733500969414 2024-12-06T16:02:52,487 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting e217722422d74343954d675c03a4b89c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1733500969799 2024-12-06T16:02:52,488 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting b72e70a06a2e4b44a790560304da054d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733500970480 2024-12-06T16:02:52,490 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d252d31b6d43033dcd4545dc84403a2e/B of d252d31b6d43033dcd4545dc84403a2e into b5616538f7284591bfd21f1d1e682ea6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:52,490 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:52,490 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., storeName=d252d31b6d43033dcd4545dc84403a2e/B, priority=13, startTime=1733500972418; duration=0sec 2024-12-06T16:02:52,490 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:52,490 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d252d31b6d43033dcd4545dc84403a2e:B 2024-12-06T16:02:52,502 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d252d31b6d43033dcd4545dc84403a2e#C#compaction#480 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:52,503 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/80515468621c406faca81a1c154ee6f0 is 50, key is test_row_0/C:col10/1733500971156/Put/seqid=0 2024-12-06T16:02:52,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501032490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:52,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742382_1558 (size=13255) 2024-12-06T16:02:52,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501032507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:52,524 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/80515468621c406faca81a1c154ee6f0 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/80515468621c406faca81a1c154ee6f0 2024-12-06T16:02:52,529 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d252d31b6d43033dcd4545dc84403a2e/C of d252d31b6d43033dcd4545dc84403a2e into 80515468621c406faca81a1c154ee6f0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:52,529 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:52,529 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., storeName=d252d31b6d43033dcd4545dc84403a2e/C, priority=13, startTime=1733500972418; duration=0sec 2024-12-06T16:02:52,529 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:52,529 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d252d31b6d43033dcd4545dc84403a2e:C 2024-12-06T16:02:52,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501032612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:52,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-06T16:02:52,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:52,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:52,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:52,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:52,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:52,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:52,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:52,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:52,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501032622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:52,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:52,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:52,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:52,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33884 deadline: 1733501032662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:52,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33922 deadline: 1733501032662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:52,667 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 85bef17d9292,46105,1733500827732 for 'test_row_1' in d252d31b6d43033dcd4545dc84403a2e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T16:02:52,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:52,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:52,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:52,668 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 85bef17d9292,46105,1733500827732 for 'test_row_1' in d252d31b6d43033dcd4545dc84403a2e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-06T16:02:52,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:52,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:52,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:52,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-06T16:02:52,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:52,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:52,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:52,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:52,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:52,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:52,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501032832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:52,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501032832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:52,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/56cca3a144274c248161467eae4a88e7 2024-12-06T16:02:52,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/c91e65c3f0d9449d90597322f6804ea6 is 50, key is test_row_0/B:col10/1733500972425/Put/seqid=0 2024-12-06T16:02:52,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742383_1559 (size=12301) 2024-12-06T16:02:52,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/c91e65c3f0d9449d90597322f6804ea6 2024-12-06T16:02:52,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/db4ca215dbe5494b9e3e237e23920daf is 50, key is test_row_0/C:col10/1733500972425/Put/seqid=0 2024-12-06T16:02:52,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742384_1560 (size=12301) 2024-12-06T16:02:52,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-06T16:02:52,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:52,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:52,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:52,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:52,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:52,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:53,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-06T16:02:53,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:53,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:53,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:53,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:53,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:53,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:53,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:53,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501033152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:53,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:53,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:53,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:53,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:53,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501033152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:53,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:53,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:53,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:53,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-06T16:02:53,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:53,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:53,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:53,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:53,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:53,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:02:53,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/db4ca215dbe5494b9e3e237e23920daf 2024-12-06T16:02:53,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/56cca3a144274c248161467eae4a88e7 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/56cca3a144274c248161467eae4a88e7 2024-12-06T16:02:53,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/56cca3a144274c248161467eae4a88e7, entries=150, sequenceid=402, filesize=12.0 K 2024-12-06T16:02:53,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/c91e65c3f0d9449d90597322f6804ea6 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/c91e65c3f0d9449d90597322f6804ea6 2024-12-06T16:02:53,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/c91e65c3f0d9449d90597322f6804ea6, entries=150, sequenceid=402, filesize=12.0 K 2024-12-06T16:02:53,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/db4ca215dbe5494b9e3e237e23920daf as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/db4ca215dbe5494b9e3e237e23920daf 2024-12-06T16:02:53,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/db4ca215dbe5494b9e3e237e23920daf, entries=150, sequenceid=402, filesize=12.0 K 2024-12-06T16:02:53,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for d252d31b6d43033dcd4545dc84403a2e in 892ms, sequenceid=402, compaction requested=false 2024-12-06T16:02:53,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:53,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-06T16:02:53,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-06T16:02:53,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:53,393 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing d252d31b6d43033dcd4545dc84403a2e 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-06T16:02:53,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/0c19852782004949af93d8ddf1718235 is 50, key is test_row_0/A:col10/1733500972505/Put/seqid=0 2024-12-06T16:02:53,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742385_1561 (size=12301) 2024-12-06T16:02:53,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:53,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. as already flushing 2024-12-06T16:02:53,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:53,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501033614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:53,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:53,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33912 deadline: 1733501033662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:53,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:53,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:53,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:53,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:53,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33944 deadline: 1733501033672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:53,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:53,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:53,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:53,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:53,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501033721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:53,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:53,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:53,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:53,806 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/0c19852782004949af93d8ddf1718235 2024-12-06T16:02:53,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/580beef7ed9c4c29874e8e88b77c0c4a is 50, key is test_row_0/B:col10/1733500972505/Put/seqid=0 2024-12-06T16:02:53,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742386_1562 (size=12301) 2024-12-06T16:02:53,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501033942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:53,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:53,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:53,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:54,220 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/580beef7ed9c4c29874e8e88b77c0c4a 2024-12-06T16:02:54,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/f702e1c5314044e9a02c9f1186913b59 is 50, key is test_row_0/C:col10/1733500972505/Put/seqid=0 2024-12-06T16:02:54,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742387_1563 (size=12301) 2024-12-06T16:02:54,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:02:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33870 deadline: 1733501034262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 2024-12-06T16:02:54,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 , the old value is region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:54,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:02:54,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., hostname=85bef17d9292,46105,1733500827732, seqNum=2 because the exception is null or not the one we care about 2024-12-06T16:02:54,591 INFO [Thread-2116 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2116. 2024-12-06T16:02:54,592 DEBUG [Thread-2116 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:02:54,592 DEBUG [Thread-2116 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:54,592 DEBUG [Thread-2116 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:54,592 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:02:54,594 INFO [Thread-2114 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2114. 2024-12-06T16:02:54,594 DEBUG [Thread-2114 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:02:54,594 DEBUG [Thread-2114 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:54,594 DEBUG [Thread-2114 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:54,594 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:02:54,599 INFO [Thread-2110 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2110. 2024-12-06T16:02:54,599 DEBUG [Thread-2110 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:02:54,599 DEBUG [Thread-2110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:54,599 DEBUG [Thread-2110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:54,599 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:02:54,603 INFO [Thread-2112 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2112. 2024-12-06T16:02:54,603 DEBUG [Thread-2112 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:02:54,603 DEBUG [Thread-2112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:54,603 DEBUG [Thread-2112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:54,603 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:02:54,605 INFO [Thread-2108 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2108. 2024-12-06T16:02:54,605 DEBUG [Thread-2108 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:02:54,605 DEBUG [Thread-2108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:54,605 DEBUG [Thread-2108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:54,605 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:02:54,651 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/f702e1c5314044e9a02c9f1186913b59 2024-12-06T16:02:54,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/0c19852782004949af93d8ddf1718235 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/0c19852782004949af93d8ddf1718235 2024-12-06T16:02:54,657 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/0c19852782004949af93d8ddf1718235, entries=150, sequenceid=421, filesize=12.0 K 2024-12-06T16:02:54,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/580beef7ed9c4c29874e8e88b77c0c4a as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/580beef7ed9c4c29874e8e88b77c0c4a 2024-12-06T16:02:54,661 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/580beef7ed9c4c29874e8e88b77c0c4a, entries=150, sequenceid=421, filesize=12.0 K 2024-12-06T16:02:54,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/f702e1c5314044e9a02c9f1186913b59 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/f702e1c5314044e9a02c9f1186913b59 2024-12-06T16:02:54,664 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/f702e1c5314044e9a02c9f1186913b59, entries=150, sequenceid=421, filesize=12.0 K 2024-12-06T16:02:54,665 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for d252d31b6d43033dcd4545dc84403a2e in 1272ms, sequenceid=421, compaction requested=true 2024-12-06T16:02:54,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:54,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:54,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-06T16:02:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-06T16:02:54,668 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-06T16:02:54,668 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4210 sec 2024-12-06T16:02:54,669 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 3.4260 sec 2024-12-06T16:02:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:54,673 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:02:54,673 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d252d31b6d43033dcd4545dc84403a2e 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-06T16:02:54,673 INFO [Thread-2103 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2103. 2024-12-06T16:02:54,673 DEBUG [Thread-2103 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:02:54,673 DEBUG [Thread-2103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:54,673 DEBUG [Thread-2103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:54,673 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:02:54,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/93123d07b82f49a69483c7830fccead7 is 50, key is test_row_0/A:col10/1733500973612/Put/seqid=0 2024-12-06T16:02:54,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742388_1564 (size=12301) 2024-12-06T16:02:54,683 INFO [Thread-2099 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2099. 2024-12-06T16:02:54,683 DEBUG [Thread-2099 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:02:54,683 DEBUG [Thread-2099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:54,683 DEBUG [Thread-2099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:54,683 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:02:54,693 INFO [Thread-2101 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2101. 2024-12-06T16:02:54,693 DEBUG [Thread-2101 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:02:54,693 INFO [Thread-2105 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2105. 2024-12-06T16:02:54,693 DEBUG [Thread-2101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:54,693 DEBUG [Thread-2105 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:02:54,693 DEBUG [Thread-2105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:54,693 DEBUG [Thread-2101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:54,693 DEBUG [Thread-2105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:54,693 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:02:54,693 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:02:54,784 INFO [Thread-2097 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2097. 2024-12-06T16:02:54,784 DEBUG [Thread-2097 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:02:54,784 DEBUG [Thread-2097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:54,784 DEBUG [Thread-2097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:54,784 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:02:55,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/93123d07b82f49a69483c7830fccead7 2024-12-06T16:02:55,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/10399c3b9af6431a8941e0772d1d8126 is 50, key is test_row_0/B:col10/1733500973612/Put/seqid=0 2024-12-06T16:02:55,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742389_1565 (size=12301) 2024-12-06T16:02:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-06T16:02:55,412 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:02:55,412 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-06T16:02:55,412 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-12-06T16:02:55,412 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-12-06T16:02:55,412 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-12-06T16:02:55,412 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-12-06T16:02:55,412 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-12-06T16:02:55,412 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-06T16:02:55,412 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-06T16:02:55,412 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1072 2024-12-06T16:02:55,412 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3216 rows 2024-12-06T16:02:55,412 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1059 2024-12-06T16:02:55,412 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3177 rows 2024-12-06T16:02:55,412 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1061 2024-12-06T16:02:55,412 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3183 rows 2024-12-06T16:02:55,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1069 2024-12-06T16:02:55,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3207 rows 2024-12-06T16:02:55,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1065 2024-12-06T16:02:55,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3190 rows 2024-12-06T16:02:55,413 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T16:02:55,413 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T16:02:55,413 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:55,413 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:55,413 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:02:55,414 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-06T16:02:55,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-06T16:02:55,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-06T16:02:55,417 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733500975416"}]},"ts":"1733500975416"} 2024-12-06T16:02:55,418 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-06T16:02:55,418 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-06T16:02:55,419 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-06T16:02:55,420 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d252d31b6d43033dcd4545dc84403a2e, UNASSIGN}] 2024-12-06T16:02:55,421 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d252d31b6d43033dcd4545dc84403a2e, UNASSIGN 2024-12-06T16:02:55,421 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=d252d31b6d43033dcd4545dc84403a2e, regionState=CLOSING, regionLocation=85bef17d9292,46105,1733500827732 2024-12-06T16:02:55,422 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d252d31b6d43033dcd4545dc84403a2e, UNASSIGN because future has completed 2024-12-06T16:02:55,423 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-06T16:02:55,423 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; CloseRegionProcedure d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732}] 2024-12-06T16:02:55,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/10399c3b9af6431a8941e0772d1d8126 2024-12-06T16:02:55,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/0302ee3f61e442cc894a218f2d978b36 is 50, key is test_row_0/C:col10/1733500973612/Put/seqid=0 2024-12-06T16:02:55,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742390_1566 (size=12301) 2024-12-06T16:02:55,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/0302ee3f61e442cc894a218f2d978b36 2024-12-06T16:02:55,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/93123d07b82f49a69483c7830fccead7 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/93123d07b82f49a69483c7830fccead7 2024-12-06T16:02:55,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/93123d07b82f49a69483c7830fccead7, entries=150, sequenceid=442, filesize=12.0 K 2024-12-06T16:02:55,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/10399c3b9af6431a8941e0772d1d8126 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/10399c3b9af6431a8941e0772d1d8126 2024-12-06T16:02:55,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/10399c3b9af6431a8941e0772d1d8126, entries=150, sequenceid=442, filesize=12.0 K 2024-12-06T16:02:55,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/0302ee3f61e442cc894a218f2d978b36 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/0302ee3f61e442cc894a218f2d978b36 2024-12-06T16:02:55,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/0302ee3f61e442cc894a218f2d978b36, entries=150, sequenceid=442, filesize=12.0 K 2024-12-06T16:02:55,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=26.84 KB/27480 for d252d31b6d43033dcd4545dc84403a2e in 841ms, sequenceid=442, compaction requested=true 2024-12-06T16:02:55,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:55,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d252d31b6d43033dcd4545dc84403a2e:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:02:55,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:55,514 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T16:02:55,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d252d31b6d43033dcd4545dc84403a2e:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:02:55,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:55,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d252d31b6d43033dcd4545dc84403a2e:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:02:55,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:55,514 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T16:02:55,515 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T16:02:55,515 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-06T16:02:55,515 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): d252d31b6d43033dcd4545dc84403a2e/A is initiating minor compaction (all files) 2024-12-06T16:02:55,515 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): d252d31b6d43033dcd4545dc84403a2e/B is initiating minor compaction (all files) 2024-12-06T16:02:55,515 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d252d31b6d43033dcd4545dc84403a2e/A in TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:55,515 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d252d31b6d43033dcd4545dc84403a2e/B in TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:55,515 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/abf4d4218ae04975a412859bcba15383, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/56cca3a144274c248161467eae4a88e7, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/0c19852782004949af93d8ddf1718235, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/93123d07b82f49a69483c7830fccead7] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp, totalSize=49.0 K 2024-12-06T16:02:55,515 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/b5616538f7284591bfd21f1d1e682ea6, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/c91e65c3f0d9449d90597322f6804ea6, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/580beef7ed9c4c29874e8e88b77c0c4a, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/10399c3b9af6431a8941e0772d1d8126] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp, totalSize=49.0 K 2024-12-06T16:02:55,516 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting abf4d4218ae04975a412859bcba15383, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733500970480 2024-12-06T16:02:55,516 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting b5616538f7284591bfd21f1d1e682ea6, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733500970480 2024-12-06T16:02:55,516 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56cca3a144274c248161467eae4a88e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733500971223 2024-12-06T16:02:55,516 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting c91e65c3f0d9449d90597322f6804ea6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733500971223 2024-12-06T16:02:55,516 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c19852782004949af93d8ddf1718235, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1733500972486 2024-12-06T16:02:55,516 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 580beef7ed9c4c29874e8e88b77c0c4a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1733500972486 2024-12-06T16:02:55,516 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93123d07b82f49a69483c7830fccead7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1733500973597 2024-12-06T16:02:55,517 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 10399c3b9af6431a8941e0772d1d8126, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1733500973597 2024-12-06T16:02:55,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-06T16:02:55,524 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d252d31b6d43033dcd4545dc84403a2e#A#compaction#489 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:55,525 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/ed60558208504f6da8b7d8c380aa8ed1 is 50, key is test_row_0/A:col10/1733500973612/Put/seqid=0 2024-12-06T16:02:55,527 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d252d31b6d43033dcd4545dc84403a2e#B#compaction#490 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:02:55,528 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/4280b14d7d3c43fbaea97ff639cd3e16 is 50, key is test_row_0/B:col10/1733500973612/Put/seqid=0 2024-12-06T16:02:55,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742391_1567 (size=13391) 2024-12-06T16:02:55,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742392_1568 (size=13391) 2024-12-06T16:02:55,575 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(122): Close d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:55,575 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-06T16:02:55,575 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1722): Closing d252d31b6d43033dcd4545dc84403a2e, disabling compactions & flushes 2024-12-06T16:02:55,575 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:55,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-06T16:02:55,828 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-06T16:02:55,934 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/ed60558208504f6da8b7d8c380aa8ed1 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/ed60558208504f6da8b7d8c380aa8ed1 2024-12-06T16:02:55,938 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d252d31b6d43033dcd4545dc84403a2e/A of d252d31b6d43033dcd4545dc84403a2e into ed60558208504f6da8b7d8c380aa8ed1(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:55,938 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:55,938 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., storeName=d252d31b6d43033dcd4545dc84403a2e/A, priority=12, startTime=1733500975514; duration=0sec 2024-12-06T16:02:55,938 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:02:55,938 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d252d31b6d43033dcd4545dc84403a2e:A 2024-12-06T16:02:55,938 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. because compaction request was cancelled 2024-12-06T16:02:55,938 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d252d31b6d43033dcd4545dc84403a2e:C 2024-12-06T16:02:55,939 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/4280b14d7d3c43fbaea97ff639cd3e16 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/4280b14d7d3c43fbaea97ff639cd3e16 2024-12-06T16:02:55,943 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d252d31b6d43033dcd4545dc84403a2e/B of d252d31b6d43033dcd4545dc84403a2e into 4280b14d7d3c43fbaea97ff639cd3e16(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:02:55,943 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d252d31b6d43033dcd4545dc84403a2e: 2024-12-06T16:02:55,943 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., storeName=d252d31b6d43033dcd4545dc84403a2e/B, priority=12, startTime=1733500975514; duration=0sec 2024-12-06T16:02:55,943 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:55,943 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:02:55,943 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d252d31b6d43033dcd4545dc84403a2e:B 2024-12-06T16:02:55,943 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:55,943 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. after waiting 0 ms 2024-12-06T16:02:55,943 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:55,943 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(2902): Flushing d252d31b6d43033dcd4545dc84403a2e 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-12-06T16:02:55,946 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/515f3fde1af6491791b3cdf421abd588 is 50, key is test_row_1/A:col10/1733500974782/Put/seqid=0 2024-12-06T16:02:55,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742393_1569 (size=9857) 2024-12-06T16:02:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-06T16:02:56,350 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/515f3fde1af6491791b3cdf421abd588 2024-12-06T16:02:56,356 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/c7e636c16fc0431fb377512b947604ce is 50, key is test_row_1/B:col10/1733500974782/Put/seqid=0 2024-12-06T16:02:56,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742394_1570 (size=9857) 2024-12-06T16:02:56,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-06T16:02:56,759 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/c7e636c16fc0431fb377512b947604ce 2024-12-06T16:02:56,766 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/d4db27aab1df4e9da761a8e99271874d is 50, key is test_row_1/C:col10/1733500974782/Put/seqid=0 2024-12-06T16:02:56,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742395_1571 (size=9857) 2024-12-06T16:02:57,170 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/d4db27aab1df4e9da761a8e99271874d 2024-12-06T16:02:57,174 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/A/515f3fde1af6491791b3cdf421abd588 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/515f3fde1af6491791b3cdf421abd588 2024-12-06T16:02:57,178 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/515f3fde1af6491791b3cdf421abd588, entries=100, sequenceid=451, filesize=9.6 K 2024-12-06T16:02:57,179 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/B/c7e636c16fc0431fb377512b947604ce as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/c7e636c16fc0431fb377512b947604ce 2024-12-06T16:02:57,182 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/c7e636c16fc0431fb377512b947604ce, entries=100, sequenceid=451, filesize=9.6 K 2024-12-06T16:02:57,182 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/.tmp/C/d4db27aab1df4e9da761a8e99271874d as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/d4db27aab1df4e9da761a8e99271874d 2024-12-06T16:02:57,185 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/d4db27aab1df4e9da761a8e99271874d, entries=100, sequenceid=451, filesize=9.6 K 2024-12-06T16:02:57,186 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for d252d31b6d43033dcd4545dc84403a2e in 1243ms, sequenceid=451, compaction requested=true 2024-12-06T16:02:57,186 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/336febd4ab0748f1837d4307d7071176, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/57a4102a08674c7c8b04c64c5aef2667, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/abb942014ea0493e9f34dfc309fafb43, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/462d6a05a5f841f3a26154c9b500fb50, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/0a9928b2041043cbaa50865de24ac4a2, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/81208f65481a49978209dd64c6b94b6f, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/4e270362e27447a4b0a7a11e1dd654d6, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/2c47771751a9454282b5c96a84c48ae3, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/675c164f691b49548b349b16ed47e451, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/55e61403ca7c4899af0052694fd2348e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/4bbf4bde630c45bea03b892416ab0d74, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/4f36894052064c189cdf735e5ba82d33, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/750ccae4409b4d80a5fdb1b5f32fcf07, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/9c3e588a2317430681205e7329416eb6, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/2dc98ccc9bc646bcabc9d1b03f17674f, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/94c7898da2a94f12af8a21b48dcc20f2, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/050f1f6b50a94a6998cf48f6540ac604, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/84f341b86b6c46bf95096a34ef48f390, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/165f091ea1de4722919a93baffdb1465, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/d459cb5ebb4e4d8586cca01d71edd37e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/f975113cb1ca4df3b36d55d7c3c6bfb6, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/78e5fd5b9c5945c68f9f8d35b55b3cd7, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/f94448ecfc3e4743856e860ec677fb45, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/f4c956ce052f438ba779994db7c018f3, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/6cf937ef57714bc79e1a32aaa496eadf, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/e1ae0a1ade5f49eea01324833130174a, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/7762f47b3578403da0733da5efcab3d0, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/19946b7fd249488fa20a44a25915ffd6, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/abf4d4218ae04975a412859bcba15383, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/56cca3a144274c248161467eae4a88e7, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/0c19852782004949af93d8ddf1718235, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/93123d07b82f49a69483c7830fccead7] to archive 2024-12-06T16:02:57,187 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T16:02:57,189 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/336febd4ab0748f1837d4307d7071176 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/336febd4ab0748f1837d4307d7071176 2024-12-06T16:02:57,189 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/57a4102a08674c7c8b04c64c5aef2667 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/57a4102a08674c7c8b04c64c5aef2667 2024-12-06T16:02:57,190 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/abb942014ea0493e9f34dfc309fafb43 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/abb942014ea0493e9f34dfc309fafb43 2024-12-06T16:02:57,191 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/462d6a05a5f841f3a26154c9b500fb50 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/462d6a05a5f841f3a26154c9b500fb50 2024-12-06T16:02:57,192 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/0a9928b2041043cbaa50865de24ac4a2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/0a9928b2041043cbaa50865de24ac4a2 2024-12-06T16:02:57,193 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/81208f65481a49978209dd64c6b94b6f to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/81208f65481a49978209dd64c6b94b6f 2024-12-06T16:02:57,195 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/4e270362e27447a4b0a7a11e1dd654d6 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/4e270362e27447a4b0a7a11e1dd654d6 2024-12-06T16:02:57,196 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/2c47771751a9454282b5c96a84c48ae3 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/2c47771751a9454282b5c96a84c48ae3 2024-12-06T16:02:57,197 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/675c164f691b49548b349b16ed47e451 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/675c164f691b49548b349b16ed47e451 2024-12-06T16:02:57,198 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/55e61403ca7c4899af0052694fd2348e to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/55e61403ca7c4899af0052694fd2348e 2024-12-06T16:02:57,199 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/4bbf4bde630c45bea03b892416ab0d74 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/4bbf4bde630c45bea03b892416ab0d74 2024-12-06T16:02:57,200 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/4f36894052064c189cdf735e5ba82d33 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/4f36894052064c189cdf735e5ba82d33 2024-12-06T16:02:57,201 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/750ccae4409b4d80a5fdb1b5f32fcf07 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/750ccae4409b4d80a5fdb1b5f32fcf07 2024-12-06T16:02:57,202 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/9c3e588a2317430681205e7329416eb6 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/9c3e588a2317430681205e7329416eb6 2024-12-06T16:02:57,203 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/2dc98ccc9bc646bcabc9d1b03f17674f to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/2dc98ccc9bc646bcabc9d1b03f17674f 2024-12-06T16:02:57,204 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/94c7898da2a94f12af8a21b48dcc20f2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/94c7898da2a94f12af8a21b48dcc20f2 2024-12-06T16:02:57,205 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/050f1f6b50a94a6998cf48f6540ac604 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/050f1f6b50a94a6998cf48f6540ac604 2024-12-06T16:02:57,207 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/84f341b86b6c46bf95096a34ef48f390 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/84f341b86b6c46bf95096a34ef48f390 2024-12-06T16:02:57,208 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/165f091ea1de4722919a93baffdb1465 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/165f091ea1de4722919a93baffdb1465 2024-12-06T16:02:57,209 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/d459cb5ebb4e4d8586cca01d71edd37e to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/d459cb5ebb4e4d8586cca01d71edd37e 2024-12-06T16:02:57,210 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/f975113cb1ca4df3b36d55d7c3c6bfb6 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/f975113cb1ca4df3b36d55d7c3c6bfb6 2024-12-06T16:02:57,211 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/78e5fd5b9c5945c68f9f8d35b55b3cd7 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/78e5fd5b9c5945c68f9f8d35b55b3cd7 2024-12-06T16:02:57,212 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/f94448ecfc3e4743856e860ec677fb45 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/f94448ecfc3e4743856e860ec677fb45 2024-12-06T16:02:57,213 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/f4c956ce052f438ba779994db7c018f3 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/f4c956ce052f438ba779994db7c018f3 2024-12-06T16:02:57,214 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/6cf937ef57714bc79e1a32aaa496eadf to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/6cf937ef57714bc79e1a32aaa496eadf 2024-12-06T16:02:57,215 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/e1ae0a1ade5f49eea01324833130174a to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/e1ae0a1ade5f49eea01324833130174a 2024-12-06T16:02:57,216 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/7762f47b3578403da0733da5efcab3d0 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/7762f47b3578403da0733da5efcab3d0 2024-12-06T16:02:57,218 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/19946b7fd249488fa20a44a25915ffd6 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/19946b7fd249488fa20a44a25915ffd6 2024-12-06T16:02:57,219 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/abf4d4218ae04975a412859bcba15383 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/abf4d4218ae04975a412859bcba15383 2024-12-06T16:02:57,220 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/56cca3a144274c248161467eae4a88e7 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/56cca3a144274c248161467eae4a88e7 2024-12-06T16:02:57,221 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/0c19852782004949af93d8ddf1718235 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/0c19852782004949af93d8ddf1718235 2024-12-06T16:02:57,222 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/93123d07b82f49a69483c7830fccead7 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/93123d07b82f49a69483c7830fccead7 2024-12-06T16:02:57,228 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/f882d547a3484cfc946739ec9afc6a9d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/a411fb3527be445ba11ce0d7646b0343, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/55137a9b2d24497688e4d5798bf8e107, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/0b235e64ac0c4ec09a60327072a06b03, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/1d9e87cc8fcb4c7588d5b9b890cbea9a, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/57f4033b4d7c490b8871ffe974be7efe, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/18514aac7aa9424aae04909b921e2e67, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/4e1f06be0f7b4035a4d7026a60c9eda1, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/d733c5d620b34f1d8bf5860bb0b03041, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/ee82634d48f8444fa01672e336affba0, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/d8152a203113426dadfd715a2ce1a414, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/9c615b268f154256b3fc73a10c148376, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/32bf68c2e8994354985ca2145b8baa74, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/6c3b92b366734bb8a7492a36df19f89e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/c4ff191b6ad743639098ffbe242d8411, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/b034e79856534fbeb7b7f866dc7f776d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/d5b4fb72d6a84dc1a872ade5270fdf6d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/18fe10b910e3426b855f7b0d7a4542a9, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/b56dfeda4f8940c78aefc0c044a2c89c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/be7760d3912b480b8d37e57a1151b390, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/83bfa520cf26417eaf0fc34e57cd2abe, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/6b1d7d1f799e478a9745331db61a3055, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/f65d8221a4974694986d99017f7cc2df, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/dd8c75f84c1c4656b3e165d94ada4834, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/1f09e6efa6ea4eb6ba03e1ee4225ffa7, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/7800e153e6744e929135da77d9b1566a, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/1d7854ca0ec749b5838153d9e3ad1adb, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/b5616538f7284591bfd21f1d1e682ea6, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/4187074220094f46a65e684ebbb45b46, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/c91e65c3f0d9449d90597322f6804ea6, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/580beef7ed9c4c29874e8e88b77c0c4a, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/10399c3b9af6431a8941e0772d1d8126] to archive 2024-12-06T16:02:57,230 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T16:02:57,232 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/f882d547a3484cfc946739ec9afc6a9d to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/f882d547a3484cfc946739ec9afc6a9d 2024-12-06T16:02:57,233 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/a411fb3527be445ba11ce0d7646b0343 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/a411fb3527be445ba11ce0d7646b0343 2024-12-06T16:02:57,234 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/55137a9b2d24497688e4d5798bf8e107 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/55137a9b2d24497688e4d5798bf8e107 2024-12-06T16:02:57,236 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/0b235e64ac0c4ec09a60327072a06b03 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/0b235e64ac0c4ec09a60327072a06b03 2024-12-06T16:02:57,238 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/1d9e87cc8fcb4c7588d5b9b890cbea9a to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/1d9e87cc8fcb4c7588d5b9b890cbea9a 2024-12-06T16:02:57,239 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/57f4033b4d7c490b8871ffe974be7efe to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/57f4033b4d7c490b8871ffe974be7efe 2024-12-06T16:02:57,240 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/18514aac7aa9424aae04909b921e2e67 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/18514aac7aa9424aae04909b921e2e67 2024-12-06T16:02:57,241 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/4e1f06be0f7b4035a4d7026a60c9eda1 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/4e1f06be0f7b4035a4d7026a60c9eda1 2024-12-06T16:02:57,242 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/d733c5d620b34f1d8bf5860bb0b03041 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/d733c5d620b34f1d8bf5860bb0b03041 2024-12-06T16:02:57,243 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/ee82634d48f8444fa01672e336affba0 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/ee82634d48f8444fa01672e336affba0 2024-12-06T16:02:57,244 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/d8152a203113426dadfd715a2ce1a414 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/d8152a203113426dadfd715a2ce1a414 2024-12-06T16:02:57,246 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/9c615b268f154256b3fc73a10c148376 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/9c615b268f154256b3fc73a10c148376 2024-12-06T16:02:57,247 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/32bf68c2e8994354985ca2145b8baa74 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/32bf68c2e8994354985ca2145b8baa74 2024-12-06T16:02:57,248 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/6c3b92b366734bb8a7492a36df19f89e to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/6c3b92b366734bb8a7492a36df19f89e 2024-12-06T16:02:57,252 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/c4ff191b6ad743639098ffbe242d8411 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/c4ff191b6ad743639098ffbe242d8411 2024-12-06T16:02:57,253 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/b034e79856534fbeb7b7f866dc7f776d to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/b034e79856534fbeb7b7f866dc7f776d 2024-12-06T16:02:57,255 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/d5b4fb72d6a84dc1a872ade5270fdf6d to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/d5b4fb72d6a84dc1a872ade5270fdf6d 2024-12-06T16:02:57,256 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/18fe10b910e3426b855f7b0d7a4542a9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/18fe10b910e3426b855f7b0d7a4542a9 2024-12-06T16:02:57,257 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/b56dfeda4f8940c78aefc0c044a2c89c to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/b56dfeda4f8940c78aefc0c044a2c89c 2024-12-06T16:02:57,260 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/be7760d3912b480b8d37e57a1151b390 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/be7760d3912b480b8d37e57a1151b390 2024-12-06T16:02:57,261 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/83bfa520cf26417eaf0fc34e57cd2abe to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/83bfa520cf26417eaf0fc34e57cd2abe 2024-12-06T16:02:57,262 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/6b1d7d1f799e478a9745331db61a3055 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/6b1d7d1f799e478a9745331db61a3055 2024-12-06T16:02:57,263 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/f65d8221a4974694986d99017f7cc2df to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/f65d8221a4974694986d99017f7cc2df 2024-12-06T16:02:57,264 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/dd8c75f84c1c4656b3e165d94ada4834 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/dd8c75f84c1c4656b3e165d94ada4834 2024-12-06T16:02:57,266 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/1f09e6efa6ea4eb6ba03e1ee4225ffa7 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/1f09e6efa6ea4eb6ba03e1ee4225ffa7 2024-12-06T16:02:57,267 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/7800e153e6744e929135da77d9b1566a to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/7800e153e6744e929135da77d9b1566a 2024-12-06T16:02:57,268 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/1d7854ca0ec749b5838153d9e3ad1adb to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/1d7854ca0ec749b5838153d9e3ad1adb 2024-12-06T16:02:57,269 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/b5616538f7284591bfd21f1d1e682ea6 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/b5616538f7284591bfd21f1d1e682ea6 2024-12-06T16:02:57,270 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/4187074220094f46a65e684ebbb45b46 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/4187074220094f46a65e684ebbb45b46 2024-12-06T16:02:57,271 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/c91e65c3f0d9449d90597322f6804ea6 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/c91e65c3f0d9449d90597322f6804ea6 2024-12-06T16:02:57,273 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/580beef7ed9c4c29874e8e88b77c0c4a to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/580beef7ed9c4c29874e8e88b77c0c4a 2024-12-06T16:02:57,274 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/10399c3b9af6431a8941e0772d1d8126 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/10399c3b9af6431a8941e0772d1d8126 2024-12-06T16:02:57,275 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/489acfa10f8d444cb6d63e348544f442, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/11c9d341002d4be1bde5f70c64b9cb3e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/385ca8459c6f468fbe6369db0febe726, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/f4899af17d6d4659a1898b71927a22f0, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/bbcd88ea25124715a6300b819c8fad7a, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/1a15b8ce0917446481175d150961c876, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/3e54f48a58c54201989dd183014d7b55, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/68c7d6e1611b4a7184eee8acbec66778, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/ffdf07e7d1534c35b9b5fccaff3cef09, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/ddbd6b08c93149799a00de37f514abde, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/4b9253fbbcb7413eafe0c56266f64aa0, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/6edd7e4e75264a93a3aff43e5f44b32f, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/f7c3572c60214675be7aaf52df0432f8, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/ae0767a37c34455fa81f229d1b6a3004, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/e811f38c539e46baaabec3c9f2dc2b22, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/89cdf2a1301441b9b10f23462fd1ba03, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/74ad6c3614a948469b25a1130442c03e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/a4cd53c0c71c4eee826ca251483290b9, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/ba8fc8d6f5d14431989a8033bb883ec2, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/fc95d6a29bb648f5a214d6e1164c3658, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/a9cefb93c23e4933915cceab606a07c9, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/c82d0d4899eb4b0fb512a838ab06e2e0, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/0996cf3d684247ebb380d76541da0456, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/1ef8a29293284370bbf9aea91fbce07c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/27f091a29b214277bd74e59abf9ad80b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/7df9251e801e4233860c60e90fe8e7e4, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/e217722422d74343954d675c03a4b89c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/b72e70a06a2e4b44a790560304da054d] to archive 2024-12-06T16:02:57,276 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T16:02:57,277 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/489acfa10f8d444cb6d63e348544f442 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/489acfa10f8d444cb6d63e348544f442 2024-12-06T16:02:57,277 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/11c9d341002d4be1bde5f70c64b9cb3e to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/11c9d341002d4be1bde5f70c64b9cb3e 2024-12-06T16:02:57,278 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/385ca8459c6f468fbe6369db0febe726 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/385ca8459c6f468fbe6369db0febe726 2024-12-06T16:02:57,279 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/f4899af17d6d4659a1898b71927a22f0 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/f4899af17d6d4659a1898b71927a22f0 2024-12-06T16:02:57,280 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/bbcd88ea25124715a6300b819c8fad7a to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/bbcd88ea25124715a6300b819c8fad7a 2024-12-06T16:02:57,281 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/1a15b8ce0917446481175d150961c876 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/1a15b8ce0917446481175d150961c876 2024-12-06T16:02:57,281 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/3e54f48a58c54201989dd183014d7b55 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/3e54f48a58c54201989dd183014d7b55 2024-12-06T16:02:57,282 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/68c7d6e1611b4a7184eee8acbec66778 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/68c7d6e1611b4a7184eee8acbec66778 2024-12-06T16:02:57,283 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/ffdf07e7d1534c35b9b5fccaff3cef09 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/ffdf07e7d1534c35b9b5fccaff3cef09 2024-12-06T16:02:57,284 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/ddbd6b08c93149799a00de37f514abde to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/ddbd6b08c93149799a00de37f514abde 2024-12-06T16:02:57,285 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/4b9253fbbcb7413eafe0c56266f64aa0 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/4b9253fbbcb7413eafe0c56266f64aa0 2024-12-06T16:02:57,286 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/6edd7e4e75264a93a3aff43e5f44b32f to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/6edd7e4e75264a93a3aff43e5f44b32f 2024-12-06T16:02:57,288 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/f7c3572c60214675be7aaf52df0432f8 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/f7c3572c60214675be7aaf52df0432f8 2024-12-06T16:02:57,289 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/ae0767a37c34455fa81f229d1b6a3004 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/ae0767a37c34455fa81f229d1b6a3004 2024-12-06T16:02:57,289 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/e811f38c539e46baaabec3c9f2dc2b22 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/e811f38c539e46baaabec3c9f2dc2b22 2024-12-06T16:02:57,290 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/89cdf2a1301441b9b10f23462fd1ba03 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/89cdf2a1301441b9b10f23462fd1ba03 2024-12-06T16:02:57,291 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/74ad6c3614a948469b25a1130442c03e to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/74ad6c3614a948469b25a1130442c03e 2024-12-06T16:02:57,292 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/a4cd53c0c71c4eee826ca251483290b9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/a4cd53c0c71c4eee826ca251483290b9 2024-12-06T16:02:57,292 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/ba8fc8d6f5d14431989a8033bb883ec2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/ba8fc8d6f5d14431989a8033bb883ec2 2024-12-06T16:02:57,293 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/fc95d6a29bb648f5a214d6e1164c3658 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/fc95d6a29bb648f5a214d6e1164c3658 2024-12-06T16:02:57,294 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/a9cefb93c23e4933915cceab606a07c9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/a9cefb93c23e4933915cceab606a07c9 2024-12-06T16:02:57,295 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/c82d0d4899eb4b0fb512a838ab06e2e0 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/c82d0d4899eb4b0fb512a838ab06e2e0 2024-12-06T16:02:57,295 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/0996cf3d684247ebb380d76541da0456 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/0996cf3d684247ebb380d76541da0456 2024-12-06T16:02:57,296 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/1ef8a29293284370bbf9aea91fbce07c to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/1ef8a29293284370bbf9aea91fbce07c 2024-12-06T16:02:57,297 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/27f091a29b214277bd74e59abf9ad80b to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/27f091a29b214277bd74e59abf9ad80b 2024-12-06T16:02:57,298 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/7df9251e801e4233860c60e90fe8e7e4 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/7df9251e801e4233860c60e90fe8e7e4 2024-12-06T16:02:57,299 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/e217722422d74343954d675c03a4b89c to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/e217722422d74343954d675c03a4b89c 2024-12-06T16:02:57,299 DEBUG [StoreCloser-TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/b72e70a06a2e4b44a790560304da054d to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/b72e70a06a2e4b44a790560304da054d 2024-12-06T16:02:57,304 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/recovered.edits/454.seqid, newMaxSeqId=454, maxSeqId=1 2024-12-06T16:02:57,305 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. 2024-12-06T16:02:57,305 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1676): Region close journal for d252d31b6d43033dcd4545dc84403a2e: Waiting for close lock at 1733500975575Running coprocessor pre-close hooks at 1733500975575Disabling compacts and flushes for region at 1733500975575Disabling writes for close at 1733500975943 (+368 ms)Obtaining lock to block concurrent updates at 1733500975943Preparing flush snapshotting stores in d252d31b6d43033dcd4545dc84403a2e at 1733500975943Finished memstore snapshotting TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1733500975944 (+1 ms)Flushing stores of TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e. at 1733500975944Flushing d252d31b6d43033dcd4545dc84403a2e/A: creating writer at 1733500975944Flushing d252d31b6d43033dcd4545dc84403a2e/A: appending metadata at 1733500975946 (+2 ms)Flushing d252d31b6d43033dcd4545dc84403a2e/A: closing flushed file at 1733500975946Flushing d252d31b6d43033dcd4545dc84403a2e/B: creating writer at 1733500976353 (+407 ms)Flushing d252d31b6d43033dcd4545dc84403a2e/B: appending metadata at 1733500976355 (+2 ms)Flushing d252d31b6d43033dcd4545dc84403a2e/B: closing flushed file at 1733500976355Flushing d252d31b6d43033dcd4545dc84403a2e/C: creating writer at 1733500976763 (+408 ms)Flushing d252d31b6d43033dcd4545dc84403a2e/C: appending metadata at 1733500976765 (+2 ms)Flushing d252d31b6d43033dcd4545dc84403a2e/C: closing flushed file at 1733500976765Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22c8464a: reopening flushed file at 1733500977174 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c958cee: reopening flushed file at 1733500977178 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3cf4caf3: reopening flushed file at 1733500977182 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for d252d31b6d43033dcd4545dc84403a2e in 1243ms, sequenceid=451, compaction requested=true at 1733500977186 (+4 ms)Writing region close event to WAL at 1733500977301 (+115 ms)Running coprocessor post-close hooks at 1733500977305 (+4 ms)Closed at 1733500977305 2024-12-06T16:02:57,307 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(157): Closed d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:57,308 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=d252d31b6d43033dcd4545dc84403a2e, regionState=CLOSED 2024-12-06T16:02:57,309 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=132, ppid=131, state=RUNNABLE, hasLock=false; CloseRegionProcedure d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 because future has completed 2024-12-06T16:02:57,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-06T16:02:57,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; CloseRegionProcedure d252d31b6d43033dcd4545dc84403a2e, server=85bef17d9292,46105,1733500827732 in 1.8860 sec 2024-12-06T16:02:57,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=131, resume processing ppid=130 2024-12-06T16:02:57,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, ppid=130, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d252d31b6d43033dcd4545dc84403a2e, UNASSIGN in 1.8910 sec 2024-12-06T16:02:57,314 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-06T16:02:57,314 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8940 sec 2024-12-06T16:02:57,315 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733500977315"}]},"ts":"1733500977315"} 2024-12-06T16:02:57,316 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-06T16:02:57,316 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-06T16:02:57,318 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9030 sec 2024-12-06T16:02:57,367 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-06T16:02:57,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-06T16:02:57,562 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:02:57,563 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-06T16:02:57,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:02:57,565 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=133, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:02:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-06T16:02:57,566 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=133, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:02:57,568 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:57,570 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A, FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B, FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C, FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/recovered.edits] 2024-12-06T16:02:57,573 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/515f3fde1af6491791b3cdf421abd588 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/515f3fde1af6491791b3cdf421abd588 2024-12-06T16:02:57,575 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/ed60558208504f6da8b7d8c380aa8ed1 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/A/ed60558208504f6da8b7d8c380aa8ed1 2024-12-06T16:02:57,577 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/4280b14d7d3c43fbaea97ff639cd3e16 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/4280b14d7d3c43fbaea97ff639cd3e16 2024-12-06T16:02:57,578 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/c7e636c16fc0431fb377512b947604ce to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/B/c7e636c16fc0431fb377512b947604ce 2024-12-06T16:02:57,582 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/0302ee3f61e442cc894a218f2d978b36 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/0302ee3f61e442cc894a218f2d978b36 2024-12-06T16:02:57,583 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/80515468621c406faca81a1c154ee6f0 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/80515468621c406faca81a1c154ee6f0 2024-12-06T16:02:57,584 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/d4db27aab1df4e9da761a8e99271874d to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/d4db27aab1df4e9da761a8e99271874d 2024-12-06T16:02:57,586 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/db4ca215dbe5494b9e3e237e23920daf to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/db4ca215dbe5494b9e3e237e23920daf 2024-12-06T16:02:57,587 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/f702e1c5314044e9a02c9f1186913b59 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/C/f702e1c5314044e9a02c9f1186913b59 2024-12-06T16:02:57,590 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/recovered.edits/454.seqid to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e/recovered.edits/454.seqid 2024-12-06T16:02:57,591 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d252d31b6d43033dcd4545dc84403a2e 2024-12-06T16:02:57,591 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-06T16:02:57,594 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=133, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:02:57,596 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-06T16:02:57,599 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-06T16:02:57,600 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=133, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:02:57,600 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-06T16:02:57,601 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733500977600"}]},"ts":"9223372036854775807"} 2024-12-06T16:02:57,603 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-06T16:02:57,603 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d252d31b6d43033dcd4545dc84403a2e, NAME => 'TestAcidGuarantees,,1733500953402.d252d31b6d43033dcd4545dc84403a2e.', STARTKEY => '', ENDKEY => ''}] 2024-12-06T16:02:57,603 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-06T16:02:57,603 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733500977603"}]},"ts":"9223372036854775807"} 2024-12-06T16:02:57,605 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-06T16:02:57,606 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=133, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:02:57,608 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 43 msec 2024-12-06T16:02:57,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-06T16:02:57,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-06T16:02:57,672 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:02:57,686 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=242 (was 240) - Thread LEAK? -, OpenFileDescriptor=455 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=506 (was 490) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8621 (was 8656) 2024-12-06T16:02:57,698 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=242, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=506, ProcessCount=11, AvailableMemoryMB=8621 2024-12-06T16:02:57,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-06T16:02:57,700 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T16:02:57,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=134, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-06T16:02:57,702 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T16:02:57,702 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:02:57,702 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 134 2024-12-06T16:02:57,703 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T16:02:57,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-06T16:02:57,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742396_1572 (size=959) 2024-12-06T16:02:57,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-06T16:02:58,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-06T16:02:58,122 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d 2024-12-06T16:02:58,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742397_1573 (size=53) 2024-12-06T16:02:58,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-06T16:02:58,528 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T16:02:58,528 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d143cebc3e2771774f1f66ab4723edf2, disabling compactions & flushes 2024-12-06T16:02:58,528 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:02:58,528 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:02:58,528 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. after waiting 0 ms 2024-12-06T16:02:58,528 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:02:58,528 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:02:58,528 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d143cebc3e2771774f1f66ab4723edf2: Waiting for close lock at 1733500978528Disabling compacts and flushes for region at 1733500978528Disabling writes for close at 1733500978528Writing region close event to WAL at 1733500978528Closed at 1733500978528 2024-12-06T16:02:58,530 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T16:02:58,530 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733500978530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733500978530"}]},"ts":"1733500978530"} 2024-12-06T16:02:58,532 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T16:02:58,533 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T16:02:58,534 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733500978533"}]},"ts":"1733500978533"} 2024-12-06T16:02:58,535 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-06T16:02:58,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d143cebc3e2771774f1f66ab4723edf2, ASSIGN}] 2024-12-06T16:02:58,537 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d143cebc3e2771774f1f66ab4723edf2, ASSIGN 2024-12-06T16:02:58,537 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d143cebc3e2771774f1f66ab4723edf2, ASSIGN; state=OFFLINE, location=85bef17d9292,46105,1733500827732; forceNewPlan=false, retain=false 2024-12-06T16:02:58,688 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=d143cebc3e2771774f1f66ab4723edf2, regionState=OPENING, regionLocation=85bef17d9292,46105,1733500827732 2024-12-06T16:02:58,690 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d143cebc3e2771774f1f66ab4723edf2, ASSIGN because future has completed 2024-12-06T16:02:58,690 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732}] 2024-12-06T16:02:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-06T16:02:58,845 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:02:58,845 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7752): Opening region: {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} 2024-12-06T16:02:58,845 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:02:58,845 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T16:02:58,845 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7794): checking encryption for d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:02:58,845 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7797): checking classloading for d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:02:58,846 INFO [StoreOpener-d143cebc3e2771774f1f66ab4723edf2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:02:58,848 INFO [StoreOpener-d143cebc3e2771774f1f66ab4723edf2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d143cebc3e2771774f1f66ab4723edf2 columnFamilyName A 2024-12-06T16:02:58,848 DEBUG [StoreOpener-d143cebc3e2771774f1f66ab4723edf2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:02:58,848 INFO [StoreOpener-d143cebc3e2771774f1f66ab4723edf2-1 {}] regionserver.HStore(327): Store=d143cebc3e2771774f1f66ab4723edf2/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T16:02:58,848 INFO [StoreOpener-d143cebc3e2771774f1f66ab4723edf2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:02:58,849 INFO [StoreOpener-d143cebc3e2771774f1f66ab4723edf2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d143cebc3e2771774f1f66ab4723edf2 columnFamilyName B 2024-12-06T16:02:58,849 DEBUG [StoreOpener-d143cebc3e2771774f1f66ab4723edf2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:02:58,850 INFO [StoreOpener-d143cebc3e2771774f1f66ab4723edf2-1 {}] regionserver.HStore(327): Store=d143cebc3e2771774f1f66ab4723edf2/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T16:02:58,850 INFO [StoreOpener-d143cebc3e2771774f1f66ab4723edf2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:02:58,851 INFO [StoreOpener-d143cebc3e2771774f1f66ab4723edf2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d143cebc3e2771774f1f66ab4723edf2 columnFamilyName C 2024-12-06T16:02:58,851 DEBUG [StoreOpener-d143cebc3e2771774f1f66ab4723edf2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:02:58,851 INFO [StoreOpener-d143cebc3e2771774f1f66ab4723edf2-1 {}] regionserver.HStore(327): Store=d143cebc3e2771774f1f66ab4723edf2/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T16:02:58,851 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1038): replaying wal for d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:02:58,852 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:02:58,852 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:02:58,853 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1048): stopping wal replay for d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:02:58,853 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1060): Cleaning up temporary data for d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:02:58,854 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T16:02:58,855 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1093): writing seq id for d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:02:58,867 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T16:02:58,867 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1114): Opened d143cebc3e2771774f1f66ab4723edf2; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75084625, jitterRate=0.11884810030460358}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T16:02:58,867 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:02:58,868 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1006): Region open journal for d143cebc3e2771774f1f66ab4723edf2: Running coprocessor pre-open hook at 1733500978845Writing region info on filesystem at 1733500978845Initializing all the Stores at 1733500978846 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733500978846Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733500978846Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733500978846Cleaning up temporary data from old regions at 1733500978853 (+7 ms)Running coprocessor post-open hooks at 1733500978867 (+14 ms)Region opened successfully at 1733500978868 (+1 ms) 2024-12-06T16:02:58,870 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., pid=136, masterSystemTime=1733500978842 2024-12-06T16:02:58,873 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:02:58,873 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:02:58,874 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=d143cebc3e2771774f1f66ab4723edf2, regionState=OPEN, openSeqNum=2, regionLocation=85bef17d9292,46105,1733500827732 2024-12-06T16:02:58,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 because future has completed 2024-12-06T16:02:58,876 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-06T16:02:58,876 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; OpenRegionProcedure d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 in 185 msec 2024-12-06T16:02:58,878 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-12-06T16:02:58,878 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d143cebc3e2771774f1f66ab4723edf2, ASSIGN in 341 msec 2024-12-06T16:02:58,878 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T16:02:58,878 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733500978878"}]},"ts":"1733500978878"} 2024-12-06T16:02:58,880 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-06T16:02:58,880 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T16:02:58,881 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1810 sec 2024-12-06T16:02:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-06T16:02:59,862 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:02:59,864 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75b3b1fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:59,864 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:02:59,864 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:02:59,865 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:02:59,865 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:02:59,865 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:02:59,865 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c799b60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:59,865 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:02:59,866 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:02:59,866 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:02:59,867 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47146, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:02:59,867 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@606f4bd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:02:59,867 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:02:59,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:02:59,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:02:59,869 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46774, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:02:59,870 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T16:02:59,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 85bef17d9292,45907,1733500826847 2024-12-06T16:02:59,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7835ff93 2024-12-06T16:02:59,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T16:02:59,871 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T16:02:59,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-06T16:02:59,873 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T16:02:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-06T16:02:59,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742398_1574 (size=995) 2024-12-06T16:03:00,286 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-12-06T16:03:00,286 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-12-06T16:03:00,288 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-06T16:03:00,290 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d143cebc3e2771774f1f66ab4723edf2, REOPEN/MOVE}] 2024-12-06T16:03:00,291 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d143cebc3e2771774f1f66ab4723edf2, REOPEN/MOVE 2024-12-06T16:03:00,292 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=d143cebc3e2771774f1f66ab4723edf2, regionState=CLOSING, regionLocation=85bef17d9292,46105,1733500827732 2024-12-06T16:03:00,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d143cebc3e2771774f1f66ab4723edf2, REOPEN/MOVE because future has completed 2024-12-06T16:03:00,293 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-06T16:03:00,293 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732}] 2024-12-06T16:03:00,445 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:00,445 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-06T16:03:00,445 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing d143cebc3e2771774f1f66ab4723edf2, disabling compactions & flushes 2024-12-06T16:03:00,445 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:00,445 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:00,446 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. after waiting 0 ms 2024-12-06T16:03:00,446 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:00,449 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-06T16:03:00,449 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:00,449 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for d143cebc3e2771774f1f66ab4723edf2: Waiting for close lock at 1733500980445Running coprocessor pre-close hooks at 1733500980445Disabling compacts and flushes for region at 1733500980445Disabling writes for close at 1733500980446 (+1 ms)Writing region close event to WAL at 1733500980446Running coprocessor post-close hooks at 1733500980449 (+3 ms)Closed at 1733500980449 2024-12-06T16:03:00,449 WARN [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionServer(3299): Not adding moved region record: d143cebc3e2771774f1f66ab4723edf2 to self. 2024-12-06T16:03:00,451 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:00,451 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=d143cebc3e2771774f1f66ab4723edf2, regionState=CLOSED 2024-12-06T16:03:00,453 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 because future has completed 2024-12-06T16:03:00,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-06T16:03:00,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 in 160 msec 2024-12-06T16:03:00,455 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d143cebc3e2771774f1f66ab4723edf2, REOPEN/MOVE; state=CLOSED, location=85bef17d9292,46105,1733500827732; forceNewPlan=false, retain=true 2024-12-06T16:03:00,606 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=d143cebc3e2771774f1f66ab4723edf2, regionState=OPENING, regionLocation=85bef17d9292,46105,1733500827732 2024-12-06T16:03:00,608 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d143cebc3e2771774f1f66ab4723edf2, REOPEN/MOVE because future has completed 2024-12-06T16:03:00,608 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732}] 2024-12-06T16:03:00,763 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:00,763 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7752): Opening region: {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} 2024-12-06T16:03:00,764 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:00,764 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T16:03:00,764 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7794): checking encryption for d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:00,764 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7797): checking classloading for d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:00,765 INFO [StoreOpener-d143cebc3e2771774f1f66ab4723edf2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:00,766 INFO [StoreOpener-d143cebc3e2771774f1f66ab4723edf2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d143cebc3e2771774f1f66ab4723edf2 columnFamilyName A 2024-12-06T16:03:00,767 DEBUG [StoreOpener-d143cebc3e2771774f1f66ab4723edf2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:00,767 INFO [StoreOpener-d143cebc3e2771774f1f66ab4723edf2-1 {}] regionserver.HStore(327): Store=d143cebc3e2771774f1f66ab4723edf2/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T16:03:00,768 INFO [StoreOpener-d143cebc3e2771774f1f66ab4723edf2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:00,768 INFO [StoreOpener-d143cebc3e2771774f1f66ab4723edf2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d143cebc3e2771774f1f66ab4723edf2 columnFamilyName B 2024-12-06T16:03:00,768 DEBUG [StoreOpener-d143cebc3e2771774f1f66ab4723edf2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:00,769 INFO [StoreOpener-d143cebc3e2771774f1f66ab4723edf2-1 {}] regionserver.HStore(327): Store=d143cebc3e2771774f1f66ab4723edf2/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T16:03:00,769 INFO [StoreOpener-d143cebc3e2771774f1f66ab4723edf2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:00,769 INFO [StoreOpener-d143cebc3e2771774f1f66ab4723edf2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d143cebc3e2771774f1f66ab4723edf2 columnFamilyName C 2024-12-06T16:03:00,769 DEBUG [StoreOpener-d143cebc3e2771774f1f66ab4723edf2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:00,770 INFO [StoreOpener-d143cebc3e2771774f1f66ab4723edf2-1 {}] regionserver.HStore(327): Store=d143cebc3e2771774f1f66ab4723edf2/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T16:03:00,770 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1038): replaying wal for d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:00,771 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:00,771 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:00,772 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1048): stopping wal replay for d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:00,772 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1060): Cleaning up temporary data for d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:00,773 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T16:03:00,774 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1093): writing seq id for d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:00,774 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1114): Opened d143cebc3e2771774f1f66ab4723edf2; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65528914, jitterRate=-0.023543089628219604}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T16:03:00,774 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:00,775 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1006): Region open journal for d143cebc3e2771774f1f66ab4723edf2: Running coprocessor pre-open hook at 1733500980764Writing region info on filesystem at 1733500980764Initializing all the Stores at 1733500980765 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733500980765Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733500980765Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733500980765Cleaning up temporary data from old regions at 1733500980772 (+7 ms)Running coprocessor post-open hooks at 1733500980774 (+2 ms)Region opened successfully at 1733500980775 (+1 ms) 2024-12-06T16:03:00,776 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., pid=141, masterSystemTime=1733500980760 2024-12-06T16:03:00,777 DEBUG [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:00,777 INFO [RS_OPEN_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:00,778 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=d143cebc3e2771774f1f66ab4723edf2, regionState=OPEN, openSeqNum=5, regionLocation=85bef17d9292,46105,1733500827732 2024-12-06T16:03:00,779 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 because future has completed 2024-12-06T16:03:00,781 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=139 2024-12-06T16:03:00,781 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 in 172 msec 2024-12-06T16:03:00,783 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-12-06T16:03:00,783 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d143cebc3e2771774f1f66ab4723edf2, REOPEN/MOVE in 491 msec 2024-12-06T16:03:00,786 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-06T16:03:00,786 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 496 msec 2024-12-06T16:03:00,788 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 912 msec 2024-12-06T16:03:00,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-06T16:03:00,789 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:03:00,789 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22842932, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:03:00,789 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:03:00,790 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:03:00,790 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:03:00,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:03:00,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:03:00,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d452d76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:03:00,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:03:00,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:03:00,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:00,792 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47162, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:03:00,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a315bed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:03:00,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c5fa63a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:03:00,793 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:03:00,793 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:03:00,794 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:03:00,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:03:00,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:03:00,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3548a0d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:03:00,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:03:00,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:03:00,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:00,795 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:03:00,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ddea2fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:03:00,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71a933d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:03:00,796 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:03:00,796 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:03:00,797 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:03:00,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:03:00,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:03:00,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64beeb6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:03:00,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:03:00,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:03:00,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:00,798 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47204, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:03:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eda3cd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:03:00,799 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66f73499, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:03:00,799 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:03:00,799 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:03:00,799 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:03:00,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:03:00,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:03:00,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@582a3cde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:03:00,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:03:00,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:03:00,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:00,801 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:03:00,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d23aaff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:03:00,802 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f42478c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:03:00,802 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:03:00,802 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:03:00,803 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:03:00,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:03:00,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:03:00,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67388729, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:03:00,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:03:00,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:03:00,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:00,804 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:03:00,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ebab9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:03:00,805 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f6185aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:03:00,805 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:03:00,805 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:03:00,806 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:03:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:03:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:03:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45fde1c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:03:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:03:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:03:00,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:00,807 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:03:00,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cbbfb77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:03:00,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@476ca966, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:03:00,808 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:03:00,808 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:03:00,809 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:03:00,809 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:03:00,809 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:03:00,809 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4897124a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:03:00,809 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:03:00,809 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:03:00,809 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:00,810 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:03:00,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38881680, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:03:00,811 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78b91db9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:03:00,811 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:03:00,811 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:03:00,812 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:03:00,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:03:00,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:03:00,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d44408e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:03:00,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:03:00,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:03:00,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:00,813 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:03:00,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c98738, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:03:00,814 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60dcf631, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:03:00,814 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:03:00,814 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:03:00,814 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:03:00,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:03:00,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:03:00,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43abab44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:03:00,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:03:00,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:03:00,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:00,816 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47310, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:03:00,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@253ac093, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:03:00,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b3d9872, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:03:00,817 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 85bef17d9292,45907,-1 for getting cluster id 2024-12-06T16:03:00,817 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T16:03:00,817 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c112d9a-bae3-44dc-8f74-41feaf983fb0' 2024-12-06T16:03:00,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T16:03:00,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c112d9a-bae3-44dc-8f74-41feaf983fb0" 2024-12-06T16:03:00,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7515bcdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:03:00,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [85bef17d9292,45907,-1] 2024-12-06T16:03:00,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T16:03:00,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:00,818 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47328, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T16:03:00,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e9b11aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T16:03:00,819 DEBUG [Thread-2546 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:03:00,819 DEBUG [Thread-2550 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:03:00,819 DEBUG [Thread-2553 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:03:00,820 DEBUG [Thread-2559 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:03:00,820 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:03:00,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:03:00,820 DEBUG [Thread-2544 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:03:00,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:03:00,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:03:00,820 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:03:00,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:03:00,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:03:00,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:03:00,821 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46784, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:03:00,821 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:03:00,822 DEBUG [Thread-2555 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:03:00,822 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:03:00,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:03:00,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:03:00,822 DEBUG [Thread-2548 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:03:00,822 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:03:00,823 DEBUG [Thread-2557 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:03:00,823 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46820, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:03:00,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:03:00,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:03:00,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5] 2024-12-06T16:03:00,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:03:00,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:03:00,824 DEBUG [Thread-2542 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:03:00,825 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:03:00,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:03:00,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:03:00,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:03:00,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5] 2024-12-06T16:03:00,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:03:00,825 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:03:00,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5] 2024-12-06T16:03:00,826 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46836, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:03:00,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5] 2024-12-06T16:03:00,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-12-06T16:03:00,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-06T16:03:00,828 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46864, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:03:00,828 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:03:00,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5] 2024-12-06T16:03:00,828 DEBUG [Thread-2561 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T16:03:00,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5] 2024-12-06T16:03:00,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=85bef17d9292,46105,1733500827732, seqNum=-1] 2024-12-06T16:03:00,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T16:03:00,829 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:03:00,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5] 2024-12-06T16:03:00,830 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5] 2024-12-06T16:03:00,830 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:03:00,831 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:03:00,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5] 2024-12-06T16:03:00,835 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T16:03:00,836 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5] 2024-12-06T16:03:00,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:00,838 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:03:00,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d143cebc3e2771774f1f66ab4723edf2 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-06T16:03:00,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206afde213d0bd54621a1682dc8b2c15606_d143cebc3e2771774f1f66ab4723edf2 is 50, key is test_row_0/A:col10/1733500980832/Put/seqid=0 2024-12-06T16:03:00,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742399_1575 (size=14594) 2024-12-06T16:03:00,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:00,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501040876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:00,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:00,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:00,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:00,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:00,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501040905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:00,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:00,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:00,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:00,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:00,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501040905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:00,911 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:00,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:00,911 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:00,911 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:00,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501040906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:00,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:00,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:00,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:00,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:00,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501040908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:00,912 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:00,912 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:00,912 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:00,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-06T16:03:00,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-06T16:03:00,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:00,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:00,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:00,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:00,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:00,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:01,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:01,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501041021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:01,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:01,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:01,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:01,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501041022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:01,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:01,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:01,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:01,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501041022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:01,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:01,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:01,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:01,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501041023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:01,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:01,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:01,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:01,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:01,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:01,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501041023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:01,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:01,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:01,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:01,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:01,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:01,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-06T16:03:01,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:01,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:01,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:01,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:01,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:01,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:01,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-06T16:03:01,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501041231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:01,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:01,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:01,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:01,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501041231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:01,235 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:01,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:01,235 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:01,235 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501041231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:01,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:01,235 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:01,236 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:01,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501041232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:01,236 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:01,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:01,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:01,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:01,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501041232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:01,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:01,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:01,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:01,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:01,281 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:01,286 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206afde213d0bd54621a1682dc8b2c15606_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206afde213d0bd54621a1682dc8b2c15606_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:01,287 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/be4ecd92684c4969bef5a96f6da86995, store: [table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:01,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/be4ecd92684c4969bef5a96f6da86995 is 175, key is test_row_0/A:col10/1733500980832/Put/seqid=0 2024-12-06T16:03:01,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-06T16:03:01,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:01,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:01,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:01,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:01,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:01,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742400_1576 (size=39549) 2024-12-06T16:03:01,302 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/be4ecd92684c4969bef5a96f6da86995 2024-12-06T16:03:01,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/2e99081ebf59449d974a64223f3838bd is 50, key is test_row_0/B:col10/1733500980832/Put/seqid=0 2024-12-06T16:03:01,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742401_1577 (size=12001) 2024-12-06T16:03:01,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/2e99081ebf59449d974a64223f3838bd 2024-12-06T16:03:01,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/59890321bc504a39b6dd32210ad7a559 is 50, key is test_row_0/C:col10/1733500980832/Put/seqid=0 2024-12-06T16:03:01,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742402_1578 (size=12001) 2024-12-06T16:03:01,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-06T16:03:01,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:01,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:01,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:01,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:01,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:01,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-06T16:03:01,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501041542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:01,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:01,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:01,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:01,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501041542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:01,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:01,544 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:01,544 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501041543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:01,544 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:01,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:01,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:01,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:01,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:01,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501041543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501041543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:01,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-06T16:03:01,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:01,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:01,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:01,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:01,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:01,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-06T16:03:01,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:01,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:01,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:01,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:01,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:01,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:01,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/59890321bc504a39b6dd32210ad7a559 2024-12-06T16:03:01,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/be4ecd92684c4969bef5a96f6da86995 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/be4ecd92684c4969bef5a96f6da86995 2024-12-06T16:03:01,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/be4ecd92684c4969bef5a96f6da86995, entries=200, sequenceid=15, filesize=38.6 K 2024-12-06T16:03:01,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/2e99081ebf59449d974a64223f3838bd as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/2e99081ebf59449d974a64223f3838bd 2024-12-06T16:03:01,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/2e99081ebf59449d974a64223f3838bd, entries=150, sequenceid=15, filesize=11.7 K 2024-12-06T16:03:01,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/59890321bc504a39b6dd32210ad7a559 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/59890321bc504a39b6dd32210ad7a559 2024-12-06T16:03:01,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/59890321bc504a39b6dd32210ad7a559, entries=150, sequenceid=15, filesize=11.7 K 2024-12-06T16:03:01,837 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for d143cebc3e2771774f1f66ab4723edf2 in 999ms, sequenceid=15, compaction requested=false 2024-12-06T16:03:01,837 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-06T16:03:01,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:01,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-06T16:03:01,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:01,908 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing d143cebc3e2771774f1f66ab4723edf2 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-06T16:03:01,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206d6f4febb6f5444e69426776311552f55_d143cebc3e2771774f1f66ab4723edf2 is 50, key is test_row_0/A:col10/1733500980873/Put/seqid=0 2024-12-06T16:03:01,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742403_1579 (size=12154) 2024-12-06T16:03:01,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-06T16:03:02,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:02,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:02,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:02,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501042063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:02,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:02,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501042063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:02,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:02,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,070 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,070 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:02,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501042069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:02,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501042069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:02,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:02,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501042070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:02,074 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,074 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,074 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:02,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:02,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501042172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:02,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:02,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501042172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:02,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:02,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:02,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:02,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501042182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:02,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:02,186 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,186 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501042182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:02,186 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:02,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:02,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:02,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501042182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:02,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:02,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:02,337 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206d6f4febb6f5444e69426776311552f55_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206d6f4febb6f5444e69426776311552f55_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:02,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/c864a0a43e884f05a1bcf6e434e62249, store: [table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:02,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/c864a0a43e884f05a1bcf6e434e62249 is 175, key is test_row_0/A:col10/1733500980873/Put/seqid=0 2024-12-06T16:03:02,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742404_1580 (size=30955) 2024-12-06T16:03:02,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:02,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501042382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:02,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:02,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501042382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:02,384 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,384 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,384 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:02,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:02,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:02,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501042392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:02,394 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,394 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,394 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:02,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:02,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501042392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:02,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:02,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501042392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:02,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:02,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:02,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:02,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501042692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:02,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:02,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501042692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:02,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:02,696 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,696 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,696 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:02,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:02,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501042702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:02,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:02,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501042702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:02,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:02,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501042702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:02,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:02,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:02,748 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/c864a0a43e884f05a1bcf6e434e62249 2024-12-06T16:03:02,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/d1a727d8f9e040ada48a6b07e19a83f8 is 50, key is test_row_0/B:col10/1733500980873/Put/seqid=0 2024-12-06T16:03:02,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742405_1581 (size=12001) 2024-12-06T16:03:02,826 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T16:03:02,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-06T16:03:03,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501043202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:03,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:03,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:03,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:03,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501043203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:03,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:03,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:03,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:03,211 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/d1a727d8f9e040ada48a6b07e19a83f8 2024-12-06T16:03:03,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/445143ec4970402b8d17435cf1ff3afc is 50, key is test_row_0/C:col10/1733500980873/Put/seqid=0 2024-12-06T16:03:03,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:03,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501043222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:03,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:03,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:03,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:03,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:03,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501043222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:03,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:03,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:03,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:03,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:03,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501043222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:03,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:03,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:03,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:03,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742406_1582 (size=12001) 2024-12-06T16:03:03,627 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/445143ec4970402b8d17435cf1ff3afc 2024-12-06T16:03:03,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/c864a0a43e884f05a1bcf6e434e62249 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/c864a0a43e884f05a1bcf6e434e62249 2024-12-06T16:03:03,641 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/c864a0a43e884f05a1bcf6e434e62249, entries=150, sequenceid=40, filesize=30.2 K 2024-12-06T16:03:03,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/d1a727d8f9e040ada48a6b07e19a83f8 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/d1a727d8f9e040ada48a6b07e19a83f8 2024-12-06T16:03:03,645 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/d1a727d8f9e040ada48a6b07e19a83f8, entries=150, sequenceid=40, filesize=11.7 K 2024-12-06T16:03:03,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/445143ec4970402b8d17435cf1ff3afc as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/445143ec4970402b8d17435cf1ff3afc 2024-12-06T16:03:03,650 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/445143ec4970402b8d17435cf1ff3afc, entries=150, sequenceid=40, filesize=11.7 K 2024-12-06T16:03:03,651 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for d143cebc3e2771774f1f66ab4723edf2 in 1743ms, sequenceid=40, compaction requested=false 2024-12-06T16:03:03,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:03,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:03,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-12-06T16:03:03,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-12-06T16:03:03,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-12-06T16:03:03,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8220 sec 2024-12-06T16:03:03,658 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 2.8310 sec 2024-12-06T16:03:04,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:04,215 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:03:04,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d143cebc3e2771774f1f66ab4723edf2 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-06T16:03:04,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206cca803cd0bf64caabaa220118512dae4_d143cebc3e2771774f1f66ab4723edf2 is 50, key is test_row_0/A:col10/1733500982067/Put/seqid=0 2024-12-06T16:03:04,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742407_1583 (size=14594) 2024-12-06T16:03:04,228 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:04,232 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206cca803cd0bf64caabaa220118512dae4_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206cca803cd0bf64caabaa220118512dae4_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:04,233 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/e7ec001b72e94b5cbe142d56acb7a3fb, store: [table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:04,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/e7ec001b72e94b5cbe142d56acb7a3fb is 175, key is test_row_0/A:col10/1733500982067/Put/seqid=0 2024-12-06T16:03:04,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742408_1584 (size=39549) 2024-12-06T16:03:04,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:04,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501044280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:04,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:04,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:04,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501044281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:04,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:04,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:04,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501044281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:04,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:04,288 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,288 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501044282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:04,288 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:04,288 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,288 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:04,288 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:04,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501044283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:04,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:04,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:04,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501044392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:04,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:04,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501044392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:04,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501044392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:04,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501044392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:04,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501044392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:04,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:04,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501044602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:04,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501044602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:04,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:04,606 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501044602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:04,606 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,606 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:04,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:04,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501044603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:04,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501044603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:04,646 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/e7ec001b72e94b5cbe142d56acb7a3fb 2024-12-06T16:03:04,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/c53a97f238e547f687bb8e278f51e9cf is 50, key is test_row_0/B:col10/1733500982067/Put/seqid=0 2024-12-06T16:03:04,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742409_1585 (size=12001) 2024-12-06T16:03:04,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:04,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501044912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:04,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501044912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:04,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:04,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501044912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:04,917 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,917 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,917 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:04,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:04,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501044912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:04,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:04,917 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:04,917 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,917 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:04,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501044912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:04,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:04,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:04,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-06T16:03:04,992 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:03:04,993 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:03:04,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-12-06T16:03:04,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-06T16:03:04,995 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:03:04,995 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:03:04,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:03:05,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/c53a97f238e547f687bb8e278f51e9cf 2024-12-06T16:03:05,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/b6254c037d0d43a2957aa64b2e7bebd3 is 50, key is test_row_0/C:col10/1733500982067/Put/seqid=0 2024-12-06T16:03:05,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742410_1586 (size=12001) 2024-12-06T16:03:05,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-06T16:03:05,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-06T16:03:05,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:05,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:05,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:05,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:05,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:05,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:05,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-06T16:03:05,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:05,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:05,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:05,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:05,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:05,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:05,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-06T16:03:05,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:05,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501045421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:05,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:05,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:05,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:05,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:05,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501045432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:05,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:05,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:05,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:05,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:05,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501045432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:05,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:05,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501045432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:05,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:05,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:05,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:05,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:05,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:05,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:05,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501045432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:05,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:05,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:05,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:05,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:05,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-06T16:03:05,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:05,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:05,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:05,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:05,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:05,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:05,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/b6254c037d0d43a2957aa64b2e7bebd3 2024-12-06T16:03:05,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/e7ec001b72e94b5cbe142d56acb7a3fb as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/e7ec001b72e94b5cbe142d56acb7a3fb 2024-12-06T16:03:05,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/e7ec001b72e94b5cbe142d56acb7a3fb, entries=200, sequenceid=53, filesize=38.6 K 2024-12-06T16:03:05,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/c53a97f238e547f687bb8e278f51e9cf as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/c53a97f238e547f687bb8e278f51e9cf 2024-12-06T16:03:05,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/c53a97f238e547f687bb8e278f51e9cf, entries=150, sequenceid=53, filesize=11.7 K 2024-12-06T16:03:05,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/b6254c037d0d43a2957aa64b2e7bebd3 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/b6254c037d0d43a2957aa64b2e7bebd3 2024-12-06T16:03:05,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/b6254c037d0d43a2957aa64b2e7bebd3, entries=150, sequenceid=53, filesize=11.7 K 2024-12-06T16:03:05,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for d143cebc3e2771774f1f66ab4723edf2 in 1267ms, sequenceid=53, compaction requested=true 2024-12-06T16:03:05,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:05,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d143cebc3e2771774f1f66ab4723edf2:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:03:05,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:05,483 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:03:05,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d143cebc3e2771774f1f66ab4723edf2:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:03:05,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:05,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d143cebc3e2771774f1f66ab4723edf2:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:03:05,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:03:05,483 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:03:05,483 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:03:05,483 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:03:05,483 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): d143cebc3e2771774f1f66ab4723edf2/B is initiating minor compaction (all files) 2024-12-06T16:03:05,483 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): d143cebc3e2771774f1f66ab4723edf2/A is initiating minor compaction (all files) 2024-12-06T16:03:05,484 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d143cebc3e2771774f1f66ab4723edf2/A in TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:05,484 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d143cebc3e2771774f1f66ab4723edf2/B in TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:05,484 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/be4ecd92684c4969bef5a96f6da86995, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/c864a0a43e884f05a1bcf6e434e62249, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/e7ec001b72e94b5cbe142d56acb7a3fb] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp, totalSize=107.5 K 2024-12-06T16:03:05,484 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/2e99081ebf59449d974a64223f3838bd, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/d1a727d8f9e040ada48a6b07e19a83f8, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/c53a97f238e547f687bb8e278f51e9cf] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp, totalSize=35.2 K 2024-12-06T16:03:05,484 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:05,484 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. files: [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/be4ecd92684c4969bef5a96f6da86995, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/c864a0a43e884f05a1bcf6e434e62249, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/e7ec001b72e94b5cbe142d56acb7a3fb] 2024-12-06T16:03:05,484 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e99081ebf59449d974a64223f3838bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733500980832 2024-12-06T16:03:05,484 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting be4ecd92684c4969bef5a96f6da86995, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733500980832 2024-12-06T16:03:05,484 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting d1a727d8f9e040ada48a6b07e19a83f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733500980862 2024-12-06T16:03:05,484 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting c864a0a43e884f05a1bcf6e434e62249, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733500980862 2024-12-06T16:03:05,485 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting c53a97f238e547f687bb8e278f51e9cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733500982059 2024-12-06T16:03:05,485 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7ec001b72e94b5cbe142d56acb7a3fb, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733500982059 2024-12-06T16:03:05,490 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:05,491 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d143cebc3e2771774f1f66ab4723edf2#B#compaction#504 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:03:05,492 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/925005f3e4cb48e58b6c9a1e02a2b779 is 50, key is test_row_0/B:col10/1733500982067/Put/seqid=0 2024-12-06T16:03:05,493 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206df8c4c6ad37e4c26a68002697ed2129d_d143cebc3e2771774f1f66ab4723edf2 store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:05,495 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206df8c4c6ad37e4c26a68002697ed2129d_d143cebc3e2771774f1f66ab4723edf2, store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:05,495 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206df8c4c6ad37e4c26a68002697ed2129d_d143cebc3e2771774f1f66ab4723edf2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:05,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742412_1588 (size=12104) 2024-12-06T16:03:05,509 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/925005f3e4cb48e58b6c9a1e02a2b779 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/925005f3e4cb48e58b6c9a1e02a2b779 2024-12-06T16:03:05,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742411_1587 (size=4469) 2024-12-06T16:03:05,513 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d143cebc3e2771774f1f66ab4723edf2#A#compaction#503 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:03:05,515 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/8e095af1dbfe4759917be065681b9c55 is 175, key is test_row_0/A:col10/1733500982067/Put/seqid=0 2024-12-06T16:03:05,515 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d143cebc3e2771774f1f66ab4723edf2/B of d143cebc3e2771774f1f66ab4723edf2 into 925005f3e4cb48e58b6c9a1e02a2b779(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:03:05,515 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:05,515 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., storeName=d143cebc3e2771774f1f66ab4723edf2/B, priority=13, startTime=1733500985483; duration=0sec 2024-12-06T16:03:05,516 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:03:05,516 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d143cebc3e2771774f1f66ab4723edf2:B 2024-12-06T16:03:05,516 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:03:05,516 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:03:05,517 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): d143cebc3e2771774f1f66ab4723edf2/C is initiating minor compaction (all files) 2024-12-06T16:03:05,517 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d143cebc3e2771774f1f66ab4723edf2/C in TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:05,517 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/59890321bc504a39b6dd32210ad7a559, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/445143ec4970402b8d17435cf1ff3afc, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/b6254c037d0d43a2957aa64b2e7bebd3] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp, totalSize=35.2 K 2024-12-06T16:03:05,517 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 59890321bc504a39b6dd32210ad7a559, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733500980832 2024-12-06T16:03:05,517 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 445143ec4970402b8d17435cf1ff3afc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733500980862 2024-12-06T16:03:05,518 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting b6254c037d0d43a2957aa64b2e7bebd3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733500982059 2024-12-06T16:03:05,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742413_1589 (size=31058) 2024-12-06T16:03:05,544 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/8e095af1dbfe4759917be065681b9c55 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/8e095af1dbfe4759917be065681b9c55 2024-12-06T16:03:05,549 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d143cebc3e2771774f1f66ab4723edf2/A of d143cebc3e2771774f1f66ab4723edf2 into 8e095af1dbfe4759917be065681b9c55(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:03:05,549 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:05,549 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., storeName=d143cebc3e2771774f1f66ab4723edf2/A, priority=13, startTime=1733500985483; duration=0sec 2024-12-06T16:03:05,549 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:05,549 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d143cebc3e2771774f1f66ab4723edf2:A 2024-12-06T16:03:05,550 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d143cebc3e2771774f1f66ab4723edf2#C#compaction#505 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:03:05,550 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/8e29eb5b071e4896b13afe1274343fe9 is 50, key is test_row_0/C:col10/1733500982067/Put/seqid=0 2024-12-06T16:03:05,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742414_1590 (size=12104) 2024-12-06T16:03:05,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-06T16:03:05,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:05,607 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing d143cebc3e2771774f1f66ab4723edf2 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-06T16:03:05,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412067c98eae8f2fd4980bc1e8b71a35c61de_d143cebc3e2771774f1f66ab4723edf2 is 50, key is test_row_0/A:col10/1733500984282/Put/seqid=0 2024-12-06T16:03:05,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742415_1591 (size=12154) 2024-12-06T16:03:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-06T16:03:05,959 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/8e29eb5b071e4896b13afe1274343fe9 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/8e29eb5b071e4896b13afe1274343fe9 2024-12-06T16:03:05,964 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d143cebc3e2771774f1f66ab4723edf2/C of d143cebc3e2771774f1f66ab4723edf2 into 8e29eb5b071e4896b13afe1274343fe9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:03:05,964 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:05,964 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., storeName=d143cebc3e2771774f1f66ab4723edf2/C, priority=13, startTime=1733500985483; duration=0sec 2024-12-06T16:03:05,964 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:05,964 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d143cebc3e2771774f1f66ab4723edf2:C 2024-12-06T16:03:06,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:06,023 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412067c98eae8f2fd4980bc1e8b71a35c61de_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412067c98eae8f2fd4980bc1e8b71a35c61de_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:06,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/aaead29bdced4206b1ac7fc9e775833c, store: [table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:06,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/aaead29bdced4206b1ac7fc9e775833c is 175, key is test_row_0/A:col10/1733500984282/Put/seqid=0 2024-12-06T16:03:06,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742416_1592 (size=30955) 2024-12-06T16:03:06,029 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/aaead29bdced4206b1ac7fc9e775833c 2024-12-06T16:03:06,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/7b6edebe472b408dacfed7db7b633ea7 is 50, key is test_row_0/B:col10/1733500984282/Put/seqid=0 2024-12-06T16:03:06,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742417_1593 (size=12001) 2024-12-06T16:03:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-06T16:03:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:06,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:06,447 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/7b6edebe472b408dacfed7db7b633ea7 2024-12-06T16:03:06,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/3aab86a02fc24b92acb08b8f921c3efb is 50, key is test_row_0/C:col10/1733500984282/Put/seqid=0 2024-12-06T16:03:06,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501046455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:06,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501046456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:06,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:06,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:06,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:06,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:06,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:06,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:06,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742418_1594 (size=12001) 2024-12-06T16:03:06,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501046461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:06,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501046461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:06,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501046461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:06,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:06,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:06,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:06,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501046572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:06,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501046572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:06,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501046572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:06,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501046572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:06,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501046572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:06,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501046782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:06,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501046782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:06,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501046782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:06,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:06,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:06,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501046782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:06,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:06,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:06,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:06,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501046783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:06,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:06,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:06,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:06,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:06,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:06,866 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/3aab86a02fc24b92acb08b8f921c3efb 2024-12-06T16:03:06,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/aaead29bdced4206b1ac7fc9e775833c as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/aaead29bdced4206b1ac7fc9e775833c 2024-12-06T16:03:06,874 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/aaead29bdced4206b1ac7fc9e775833c, entries=150, sequenceid=78, filesize=30.2 K 2024-12-06T16:03:06,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/7b6edebe472b408dacfed7db7b633ea7 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/7b6edebe472b408dacfed7db7b633ea7 2024-12-06T16:03:06,878 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/7b6edebe472b408dacfed7db7b633ea7, entries=150, sequenceid=78, filesize=11.7 K 2024-12-06T16:03:06,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/3aab86a02fc24b92acb08b8f921c3efb as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/3aab86a02fc24b92acb08b8f921c3efb 2024-12-06T16:03:06,883 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/3aab86a02fc24b92acb08b8f921c3efb, entries=150, sequenceid=78, filesize=11.7 K 2024-12-06T16:03:06,883 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=80.51 KB/82440 for d143cebc3e2771774f1f66ab4723edf2 in 1276ms, sequenceid=78, compaction requested=false 2024-12-06T16:03:06,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:06,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:06,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-12-06T16:03:06,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-12-06T16:03:06,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-12-06T16:03:06,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8890 sec 2024-12-06T16:03:06,891 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 1.8960 sec 2024-12-06T16:03:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:07,094 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:03:07,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d143cebc3e2771774f1f66ab4723edf2 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-06T16:03:07,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120661e9a7cc52164cca91b4be508f53d9cf_d143cebc3e2771774f1f66ab4723edf2 is 50, key is test_row_0/A:col10/1733500986456/Put/seqid=0 2024-12-06T16:03:07,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742419_1595 (size=12154) 2024-12-06T16:03:07,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:07,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501047114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:07,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:07,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501047115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:07,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:07,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:07,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501047115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:07,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:07,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:07,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:07,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501047119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501047120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:07,124 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,124 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,124 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:07,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-06T16:03:07,142 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:03:07,143 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:03:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-12-06T16:03:07,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-06T16:03:07,145 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:03:07,145 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:03:07,146 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:03:07,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501047232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:07,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501047232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:07,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501047232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:07,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501047233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:07,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501047233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:07,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:07,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-06T16:03:07,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-06T16:03:07,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:07,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:07,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:07,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:07,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:07,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:07,366 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-06T16:03:07,367 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-06T16:03:07,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:07,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501047441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:07,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:07,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501047441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:07,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:07,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:07,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:07,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501047441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:07,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:07,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:07,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501047441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:07,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:07,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501047442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:07,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:07,446 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,447 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,447 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:07,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-06T16:03:07,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:07,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:07,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:07,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:07,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:07,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:07,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-06T16:03:07,507 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:07,510 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120661e9a7cc52164cca91b4be508f53d9cf_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120661e9a7cc52164cca91b4be508f53d9cf_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:07,511 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/692ec80c1d0542159b97a9c29eb85720, store: [table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:07,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/692ec80c1d0542159b97a9c29eb85720 is 175, key is test_row_0/A:col10/1733500986456/Put/seqid=0 2024-12-06T16:03:07,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742420_1596 (size=30955) 2024-12-06T16:03:07,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-06T16:03:07,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:07,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:07,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:07,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:07,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:07,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:07,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:07,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501047752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:07,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:07,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501047752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:07,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:07,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:07,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501047752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:07,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:07,756 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,756 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,756 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:07,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:07,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501047753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:07,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:07,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501047753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:07,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:07,757 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,757 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:07,757 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:07,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-06T16:03:07,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:07,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:07,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:07,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:07,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:07,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:07,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-06T16:03:07,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-06T16:03:07,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:07,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:07,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:07,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:07,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:07,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:07,931 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/692ec80c1d0542159b97a9c29eb85720 2024-12-06T16:03:07,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/3f87e4357ca54f9683339da93e460820 is 50, key is test_row_0/B:col10/1733500986456/Put/seqid=0 2024-12-06T16:03:07,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742421_1597 (size=12001) 2024-12-06T16:03:07,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/3f87e4357ca54f9683339da93e460820 2024-12-06T16:03:07,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/cf40a81d832b493193f66bf0fd54a2d4 is 50, key is test_row_0/C:col10/1733500986456/Put/seqid=0 2024-12-06T16:03:07,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742422_1598 (size=12001) 2024-12-06T16:03:07,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/cf40a81d832b493193f66bf0fd54a2d4 2024-12-06T16:03:07,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/692ec80c1d0542159b97a9c29eb85720 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/692ec80c1d0542159b97a9c29eb85720 2024-12-06T16:03:07,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/692ec80c1d0542159b97a9c29eb85720, entries=150, sequenceid=95, filesize=30.2 K 2024-12-06T16:03:07,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/3f87e4357ca54f9683339da93e460820 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/3f87e4357ca54f9683339da93e460820 2024-12-06T16:03:07,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/3f87e4357ca54f9683339da93e460820, entries=150, sequenceid=95, filesize=11.7 K 2024-12-06T16:03:07,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/cf40a81d832b493193f66bf0fd54a2d4 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/cf40a81d832b493193f66bf0fd54a2d4 2024-12-06T16:03:08,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/cf40a81d832b493193f66bf0fd54a2d4, entries=150, sequenceid=95, filesize=11.7 K 2024-12-06T16:03:08,001 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for d143cebc3e2771774f1f66ab4723edf2 in 907ms, sequenceid=95, compaction requested=true 2024-12-06T16:03:08,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:08,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d143cebc3e2771774f1f66ab4723edf2:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:03:08,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:08,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d143cebc3e2771774f1f66ab4723edf2:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:03:08,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:08,001 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:03:08,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d143cebc3e2771774f1f66ab4723edf2:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:03:08,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:03:08,001 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:03:08,002 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:03:08,002 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:03:08,002 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): d143cebc3e2771774f1f66ab4723edf2/B is initiating minor compaction (all files) 2024-12-06T16:03:08,002 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): d143cebc3e2771774f1f66ab4723edf2/A is initiating minor compaction (all files) 2024-12-06T16:03:08,002 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d143cebc3e2771774f1f66ab4723edf2/A in TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:08,003 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/8e095af1dbfe4759917be065681b9c55, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/aaead29bdced4206b1ac7fc9e775833c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/692ec80c1d0542159b97a9c29eb85720] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp, totalSize=90.8 K 2024-12-06T16:03:08,003 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:08,003 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. files: [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/8e095af1dbfe4759917be065681b9c55, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/aaead29bdced4206b1ac7fc9e775833c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/692ec80c1d0542159b97a9c29eb85720] 2024-12-06T16:03:08,003 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d143cebc3e2771774f1f66ab4723edf2/B in TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:08,003 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/925005f3e4cb48e58b6c9a1e02a2b779, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/7b6edebe472b408dacfed7db7b633ea7, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/3f87e4357ca54f9683339da93e460820] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp, totalSize=35.3 K 2024-12-06T16:03:08,003 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e095af1dbfe4759917be065681b9c55, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733500982059 2024-12-06T16:03:08,003 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 925005f3e4cb48e58b6c9a1e02a2b779, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733500982059 2024-12-06T16:03:08,004 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b6edebe472b408dacfed7db7b633ea7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733500984274 2024-12-06T16:03:08,004 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting aaead29bdced4206b1ac7fc9e775833c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733500984274 2024-12-06T16:03:08,004 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 692ec80c1d0542159b97a9c29eb85720, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733500986454 2024-12-06T16:03:08,004 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f87e4357ca54f9683339da93e460820, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733500986454 2024-12-06T16:03:08,012 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d143cebc3e2771774f1f66ab4723edf2#B#compaction#512 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:03:08,013 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/fd737c3639e74ff7b0821b83a8662d1d is 50, key is test_row_0/B:col10/1733500986456/Put/seqid=0 2024-12-06T16:03:08,014 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:08,016 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120669b5cc521f9a41b3bafd8d5665e6d721_d143cebc3e2771774f1f66ab4723edf2 store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:08,018 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120669b5cc521f9a41b3bafd8d5665e6d721_d143cebc3e2771774f1f66ab4723edf2, store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:08,018 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120669b5cc521f9a41b3bafd8d5665e6d721_d143cebc3e2771774f1f66ab4723edf2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:08,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742424_1600 (size=4469) 2024-12-06T16:03:08,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742423_1599 (size=12207) 2024-12-06T16:03:08,046 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d143cebc3e2771774f1f66ab4723edf2#A#compaction#513 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:03:08,047 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/111bfa18bf7148d8bf08efbabc5e4fa3 is 175, key is test_row_0/A:col10/1733500986456/Put/seqid=0 2024-12-06T16:03:08,052 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/fd737c3639e74ff7b0821b83a8662d1d as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/fd737c3639e74ff7b0821b83a8662d1d 2024-12-06T16:03:08,058 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d143cebc3e2771774f1f66ab4723edf2/B of d143cebc3e2771774f1f66ab4723edf2 into fd737c3639e74ff7b0821b83a8662d1d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:03:08,058 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:08,058 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., storeName=d143cebc3e2771774f1f66ab4723edf2/B, priority=13, startTime=1733500988001; duration=0sec 2024-12-06T16:03:08,058 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:03:08,058 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d143cebc3e2771774f1f66ab4723edf2:B 2024-12-06T16:03:08,058 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:03:08,059 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:03:08,059 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): d143cebc3e2771774f1f66ab4723edf2/C is initiating minor compaction (all files) 2024-12-06T16:03:08,059 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d143cebc3e2771774f1f66ab4723edf2/C in TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:08,059 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/8e29eb5b071e4896b13afe1274343fe9, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/3aab86a02fc24b92acb08b8f921c3efb, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/cf40a81d832b493193f66bf0fd54a2d4] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp, totalSize=35.3 K 2024-12-06T16:03:08,060 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e29eb5b071e4896b13afe1274343fe9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733500982059 2024-12-06T16:03:08,061 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 3aab86a02fc24b92acb08b8f921c3efb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733500984274 2024-12-06T16:03:08,061 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting cf40a81d832b493193f66bf0fd54a2d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733500986454 2024-12-06T16:03:08,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-06T16:03:08,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:08,063 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing d143cebc3e2771774f1f66ab4723edf2 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-12-06T16:03:08,071 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d143cebc3e2771774f1f66ab4723edf2#C#compaction#514 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:03:08,071 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/8725968be9274f3f878127aed2212a5c is 50, key is test_row_0/C:col10/1733500986456/Put/seqid=0 2024-12-06T16:03:08,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120625913b0085e8445bbb2146646dfd527f_d143cebc3e2771774f1f66ab4723edf2 is 50, key is test_row_0/A:col10/1733500987117/Put/seqid=0 2024-12-06T16:03:08,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742425_1601 (size=31161) 2024-12-06T16:03:08,081 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/111bfa18bf7148d8bf08efbabc5e4fa3 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/111bfa18bf7148d8bf08efbabc5e4fa3 2024-12-06T16:03:08,087 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d143cebc3e2771774f1f66ab4723edf2/A of d143cebc3e2771774f1f66ab4723edf2 into 111bfa18bf7148d8bf08efbabc5e4fa3(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:03:08,087 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:08,087 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., storeName=d143cebc3e2771774f1f66ab4723edf2/A, priority=13, startTime=1733500988001; duration=0sec 2024-12-06T16:03:08,087 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:08,087 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d143cebc3e2771774f1f66ab4723edf2:A 2024-12-06T16:03:08,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742426_1602 (size=12207) 2024-12-06T16:03:08,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742427_1603 (size=12154) 2024-12-06T16:03:08,097 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/8725968be9274f3f878127aed2212a5c as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/8725968be9274f3f878127aed2212a5c 2024-12-06T16:03:08,105 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d143cebc3e2771774f1f66ab4723edf2/C of d143cebc3e2771774f1f66ab4723edf2 into 8725968be9274f3f878127aed2212a5c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:03:08,105 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:08,105 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., storeName=d143cebc3e2771774f1f66ab4723edf2/C, priority=13, startTime=1733500988001; duration=0sec 2024-12-06T16:03:08,105 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:08,105 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d143cebc3e2771774f1f66ab4723edf2:C 2024-12-06T16:03:08,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:08,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:08,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-06T16:03:08,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:08,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501048277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:08,282 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,282 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,282 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:08,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:08,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501048279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:08,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501048280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:08,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:08,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:08,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501048281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:08,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:08,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:08,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501048283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:08,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:08,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:08,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501048392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:08,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:08,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501048392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:08,396 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:08,396 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:08,396 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:08,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501048392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:08,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501048392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:08,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:08,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:08,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:08,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501048392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:08,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:08,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:08,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,498 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120625913b0085e8445bbb2146646dfd527f_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120625913b0085e8445bbb2146646dfd527f_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:08,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/a2e7344100864d2189827808653ba5f8, store: [table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:08,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/a2e7344100864d2189827808653ba5f8 is 175, key is test_row_0/A:col10/1733500987117/Put/seqid=0 2024-12-06T16:03:08,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742428_1604 (size=30955) 2024-12-06T16:03:08,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:08,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501048602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:08,604 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:08,604 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,604 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:08,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501048602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:08,604 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,604 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:08,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:08,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501048602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:08,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:08,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:08,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501048602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:08,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:08,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:08,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501048602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:08,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:08,914 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/a2e7344100864d2189827808653ba5f8 2024-12-06T16:03:08,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:08,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501048912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:08,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501048912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:08,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:08,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:08,915 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,915 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501048912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:08,915 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:08,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501048912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:08,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:08,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501048913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:08,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:08,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:08,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:08,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:08,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/2a1db293552e4b3e9f74c5232377cdd1 is 50, key is test_row_0/B:col10/1733500987117/Put/seqid=0 2024-12-06T16:03:08,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742429_1605 (size=12001) 2024-12-06T16:03:08,931 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/2a1db293552e4b3e9f74c5232377cdd1 2024-12-06T16:03:08,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/89ea38abeb1f4f349422c256284058c5 is 50, key is test_row_0/C:col10/1733500987117/Put/seqid=0 2024-12-06T16:03:08,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742430_1606 (size=12001) 2024-12-06T16:03:08,944 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/89ea38abeb1f4f349422c256284058c5 2024-12-06T16:03:08,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/a2e7344100864d2189827808653ba5f8 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/a2e7344100864d2189827808653ba5f8 2024-12-06T16:03:08,953 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/a2e7344100864d2189827808653ba5f8, entries=150, sequenceid=116, filesize=30.2 K 2024-12-06T16:03:08,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/2a1db293552e4b3e9f74c5232377cdd1 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/2a1db293552e4b3e9f74c5232377cdd1 2024-12-06T16:03:08,958 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/2a1db293552e4b3e9f74c5232377cdd1, entries=150, sequenceid=116, filesize=11.7 K 2024-12-06T16:03:08,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/89ea38abeb1f4f349422c256284058c5 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/89ea38abeb1f4f349422c256284058c5 2024-12-06T16:03:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,963 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/89ea38abeb1f4f349422c256284058c5, entries=150, sequenceid=116, filesize=11.7 K 2024-12-06T16:03:08,964 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for d143cebc3e2771774f1f66ab4723edf2 in 900ms, sequenceid=116, compaction requested=false 2024-12-06T16:03:08,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:08,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:08,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-12-06T16:03:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-12-06T16:03:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-06T16:03:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8200 sec 2024-12-06T16:03:08,969 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.8250 sec 2024-12-06T16:03:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-06T16:03:09,296 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:03:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,297 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:03:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-12-06T16:03:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-06T16:03:09,312 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:03:09,313 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:03:09,313 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:03:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-06T16:03:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:09,427 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:03:09,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d143cebc3e2771774f1f66ab4723edf2 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-12-06T16:03:09,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412060db15f40de014e8098267313c7328f70_d143cebc3e2771774f1f66ab4723edf2 is 50, key is test_row_0/A:col10/1733500989425/Put/seqid=0 2024-12-06T16:03:09,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742431_1607 (size=9814) 2024-12-06T16:03:09,451 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:09,455 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412060db15f40de014e8098267313c7328f70_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412060db15f40de014e8098267313c7328f70_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:09,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:09,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501049448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:09,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:09,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501049450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:09,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:09,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501049451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:09,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:09,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:09,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:09,457 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/ea053a8a453d4ae194ef4970860b3aa6, store: [table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:09,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:09,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:09,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:09,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:09,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:09,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501049451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:09,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:09,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:09,458 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:09,458 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:09,458 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:09,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/ea053a8a453d4ae194ef4970860b3aa6 is 175, key is test_row_0/A:col10/1733500989425/Put/seqid=0 2024-12-06T16:03:09,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:09,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501049457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:09,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:09,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:09,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:09,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742432_1608 (size=22461) 2024-12-06T16:03:09,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-06T16:03:09,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:09,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:09,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:09,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:09,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:09,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:09,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501049562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:09,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:09,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:09,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:09,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501049562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:09,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501049562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:09,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:09,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501049562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:09,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:09,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:09,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501049562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:09,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:09,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:09,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:09,568 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:09,568 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:09,568 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:09,568 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:09,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:09,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-06T16:03:09,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-06T16:03:09,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:09,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:09,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:09,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:09,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:09,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:09,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:09,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501049772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:09,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:09,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:09,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:09,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:09,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501049772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:09,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:09,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:09,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:09,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:09,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501049772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:09,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:09,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:09,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:09,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501049772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:09,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:09,776 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:09,776 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:09,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-06T16:03:09,776 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:09,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:09,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:09,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:09,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:09,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:09,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:09,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:09,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501049782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:09,782 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:09,782 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:09,782 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:09,866 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/ea053a8a453d4ae194ef4970860b3aa6 2024-12-06T16:03:09,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/b03a88f93a9846d4a40301788b0cc79c is 50, key is test_row_0/B:col10/1733500989425/Put/seqid=0 2024-12-06T16:03:09,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742433_1609 (size=9757) 2024-12-06T16:03:09,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/b03a88f93a9846d4a40301788b0cc79c 2024-12-06T16:03:09,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/744e1ed3c33f4193a75bfe7f2c402132 is 50, key is test_row_0/C:col10/1733500989425/Put/seqid=0 2024-12-06T16:03:09,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742434_1610 (size=9757) 2024-12-06T16:03:09,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/744e1ed3c33f4193a75bfe7f2c402132 2024-12-06T16:03:09,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/ea053a8a453d4ae194ef4970860b3aa6 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/ea053a8a453d4ae194ef4970860b3aa6 2024-12-06T16:03:09,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/ea053a8a453d4ae194ef4970860b3aa6, entries=100, sequenceid=136, filesize=21.9 K 2024-12-06T16:03:09,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-06T16:03:09,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:09,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:09,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:09,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:09,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:09,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-06T16:03:09,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:09,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/b03a88f93a9846d4a40301788b0cc79c as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/b03a88f93a9846d4a40301788b0cc79c 2024-12-06T16:03:09,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/b03a88f93a9846d4a40301788b0cc79c, entries=100, sequenceid=136, filesize=9.5 K 2024-12-06T16:03:09,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/744e1ed3c33f4193a75bfe7f2c402132 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/744e1ed3c33f4193a75bfe7f2c402132 2024-12-06T16:03:09,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/744e1ed3c33f4193a75bfe7f2c402132, entries=100, sequenceid=136, filesize=9.5 K 2024-12-06T16:03:09,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for d143cebc3e2771774f1f66ab4723edf2 in 522ms, sequenceid=136, compaction requested=true 2024-12-06T16:03:09,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:09,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d143cebc3e2771774f1f66ab4723edf2:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:03:09,950 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:03:09,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:09,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d143cebc3e2771774f1f66ab4723edf2:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:03:09,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:09,950 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:03:09,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d143cebc3e2771774f1f66ab4723edf2:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:03:09,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:03:09,951 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 84577 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:03:09,952 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): d143cebc3e2771774f1f66ab4723edf2/A is initiating minor compaction (all files) 2024-12-06T16:03:09,952 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d143cebc3e2771774f1f66ab4723edf2/A in TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:09,952 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/111bfa18bf7148d8bf08efbabc5e4fa3, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/a2e7344100864d2189827808653ba5f8, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/ea053a8a453d4ae194ef4970860b3aa6] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp, totalSize=82.6 K 2024-12-06T16:03:09,952 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:09,952 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. files: [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/111bfa18bf7148d8bf08efbabc5e4fa3, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/a2e7344100864d2189827808653ba5f8, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/ea053a8a453d4ae194ef4970860b3aa6] 2024-12-06T16:03:09,952 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:03:09,952 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): d143cebc3e2771774f1f66ab4723edf2/B is initiating minor compaction (all files) 2024-12-06T16:03:09,952 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d143cebc3e2771774f1f66ab4723edf2/B in TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:09,952 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/fd737c3639e74ff7b0821b83a8662d1d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/2a1db293552e4b3e9f74c5232377cdd1, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/b03a88f93a9846d4a40301788b0cc79c] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp, totalSize=33.2 K 2024-12-06T16:03:09,953 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 111bfa18bf7148d8bf08efbabc5e4fa3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733500986454 2024-12-06T16:03:09,953 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting fd737c3639e74ff7b0821b83a8662d1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733500986454 2024-12-06T16:03:09,953 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2e7344100864d2189827808653ba5f8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733500987112 2024-12-06T16:03:09,953 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a1db293552e4b3e9f74c5232377cdd1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733500987112 2024-12-06T16:03:09,953 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea053a8a453d4ae194ef4970860b3aa6, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733500989425 2024-12-06T16:03:09,954 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting b03a88f93a9846d4a40301788b0cc79c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733500989425 2024-12-06T16:03:09,970 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:09,971 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d143cebc3e2771774f1f66ab4723edf2#B#compaction#521 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:03:09,971 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/eb1171617e084072b84c7260daaeb58e is 50, key is test_row_0/B:col10/1733500989425/Put/seqid=0 2024-12-06T16:03:09,974 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206fc06d39545c4449c983bd37e427c33bc_d143cebc3e2771774f1f66ab4723edf2 store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:09,975 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206fc06d39545c4449c983bd37e427c33bc_d143cebc3e2771774f1f66ab4723edf2, store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:09,976 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206fc06d39545c4449c983bd37e427c33bc_d143cebc3e2771774f1f66ab4723edf2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:09,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742435_1611 (size=12409) 2024-12-06T16:03:09,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742436_1612 (size=4469) 2024-12-06T16:03:09,998 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d143cebc3e2771774f1f66ab4723edf2#A#compaction#522 average throughput is 0.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:03:09,999 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/e5cd51d8e5f2487cae978b7121f0c86b is 175, key is test_row_0/A:col10/1733500989425/Put/seqid=0 2024-12-06T16:03:10,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742437_1613 (size=31470) 2024-12-06T16:03:10,013 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/e5cd51d8e5f2487cae978b7121f0c86b as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/e5cd51d8e5f2487cae978b7121f0c86b 2024-12-06T16:03:10,022 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d143cebc3e2771774f1f66ab4723edf2/A of d143cebc3e2771774f1f66ab4723edf2 into e5cd51d8e5f2487cae978b7121f0c86b(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:03:10,022 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:10,022 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., storeName=d143cebc3e2771774f1f66ab4723edf2/A, priority=13, startTime=1733500989949; duration=0sec 2024-12-06T16:03:10,022 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:03:10,022 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d143cebc3e2771774f1f66ab4723edf2:A 2024-12-06T16:03:10,022 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:03:10,024 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:03:10,024 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): d143cebc3e2771774f1f66ab4723edf2/C is initiating minor compaction (all files) 2024-12-06T16:03:10,024 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d143cebc3e2771774f1f66ab4723edf2/C in TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:10,024 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/8725968be9274f3f878127aed2212a5c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/89ea38abeb1f4f349422c256284058c5, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/744e1ed3c33f4193a75bfe7f2c402132] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp, totalSize=33.2 K 2024-12-06T16:03:10,024 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8725968be9274f3f878127aed2212a5c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733500986454 2024-12-06T16:03:10,025 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89ea38abeb1f4f349422c256284058c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733500987112 2024-12-06T16:03:10,025 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 744e1ed3c33f4193a75bfe7f2c402132, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733500989425 2024-12-06T16:03:10,041 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d143cebc3e2771774f1f66ab4723edf2#C#compaction#523 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:03:10,041 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/a5d3927b7ae84dac9ca3f4c8df4f2129 is 50, key is test_row_0/C:col10/1733500989425/Put/seqid=0 2024-12-06T16:03:10,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742438_1614 (size=12409) 2024-12-06T16:03:10,077 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/a5d3927b7ae84dac9ca3f4c8df4f2129 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/a5d3927b7ae84dac9ca3f4c8df4f2129 2024-12-06T16:03:10,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-06T16:03:10,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:10,084 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d143cebc3e2771774f1f66ab4723edf2/C of d143cebc3e2771774f1f66ab4723edf2 into a5d3927b7ae84dac9ca3f4c8df4f2129(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:03:10,084 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:10,084 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing d143cebc3e2771774f1f66ab4723edf2 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-12-06T16:03:10,084 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., storeName=d143cebc3e2771774f1f66ab4723edf2/C, priority=13, startTime=1733500989950; duration=0sec 2024-12-06T16:03:10,085 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:10,085 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d143cebc3e2771774f1f66ab4723edf2:C 2024-12-06T16:03:10,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:10,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:10,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:10,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501050101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:10,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:10,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:10,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501050104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:10,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:10,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501050105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:10,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:10,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501050107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:10,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:10,114 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,114 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501050108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:10,114 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:10,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:10,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412064719fd9d5c1845d8a7ca527998902247_d143cebc3e2771774f1f66ab4723edf2 is 50, key is test_row_0/A:col10/1733500990083/Put/seqid=0 2024-12-06T16:03:10,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742439_1615 (size=12304) 2024-12-06T16:03:10,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:10,145 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412064719fd9d5c1845d8a7ca527998902247_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412064719fd9d5c1845d8a7ca527998902247_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:10,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/d420dae89c504f209c3cc38949e29859, store: [table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:10,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/d420dae89c504f209c3cc38949e29859 is 175, key is test_row_0/A:col10/1733500990083/Put/seqid=0 2024-12-06T16:03:10,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742440_1616 (size=31105) 2024-12-06T16:03:10,182 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/d420dae89c504f209c3cc38949e29859 2024-12-06T16:03:10,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/14442bf624344c27bfd2255ae5fe7cc8 is 50, key is test_row_0/B:col10/1733500990083/Put/seqid=0 2024-12-06T16:03:10,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:10,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501050212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:10,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:10,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501050213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:10,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:10,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:10,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501050222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:10,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:10,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501050222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:10,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:10,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501050222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:10,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742441_1617 (size=12151) 2024-12-06T16:03:10,396 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/eb1171617e084072b84c7260daaeb58e as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/eb1171617e084072b84c7260daaeb58e 2024-12-06T16:03:10,401 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d143cebc3e2771774f1f66ab4723edf2/B of d143cebc3e2771774f1f66ab4723edf2 into eb1171617e084072b84c7260daaeb58e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:03:10,401 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:10,402 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., storeName=d143cebc3e2771774f1f66ab4723edf2/B, priority=13, startTime=1733500989950; duration=0sec 2024-12-06T16:03:10,402 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:10,402 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d143cebc3e2771774f1f66ab4723edf2:B 2024-12-06T16:03:10,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:10,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501050422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:10,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:10,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501050422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:10,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:10,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:10,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501050432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:10,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501050432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:10,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:10,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501050433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:10,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-06T16:03:10,635 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/14442bf624344c27bfd2255ae5fe7cc8 2024-12-06T16:03:10,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/820b4be98bf74f039b00f78e56e9f1b2 is 50, key is test_row_0/C:col10/1733500990083/Put/seqid=0 2024-12-06T16:03:10,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742442_1618 (size=12151) 2024-12-06T16:03:10,651 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/820b4be98bf74f039b00f78e56e9f1b2 2024-12-06T16:03:10,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/d420dae89c504f209c3cc38949e29859 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/d420dae89c504f209c3cc38949e29859 2024-12-06T16:03:10,662 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/d420dae89c504f209c3cc38949e29859, entries=150, sequenceid=158, filesize=30.4 K 2024-12-06T16:03:10,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/14442bf624344c27bfd2255ae5fe7cc8 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/14442bf624344c27bfd2255ae5fe7cc8 2024-12-06T16:03:10,669 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/14442bf624344c27bfd2255ae5fe7cc8, entries=150, sequenceid=158, filesize=11.9 K 2024-12-06T16:03:10,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/820b4be98bf74f039b00f78e56e9f1b2 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/820b4be98bf74f039b00f78e56e9f1b2 2024-12-06T16:03:10,691 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/820b4be98bf74f039b00f78e56e9f1b2, entries=150, sequenceid=158, filesize=11.9 K 2024-12-06T16:03:10,692 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for d143cebc3e2771774f1f66ab4723edf2 in 608ms, sequenceid=158, compaction requested=false 2024-12-06T16:03:10,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:10,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:10,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-12-06T16:03:10,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-12-06T16:03:10,698 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-12-06T16:03:10,698 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3820 sec 2024-12-06T16:03:10,703 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.4030 sec 2024-12-06T16:03:10,735 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:03:10,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d143cebc3e2771774f1f66ab4723edf2 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-12-06T16:03:10,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:10,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206f11c441ade244e4c9dc1fd369d9b8a87_d143cebc3e2771774f1f66ab4723edf2 is 50, key is test_row_0/A:col10/1733500990106/Put/seqid=0 2024-12-06T16:03:10,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742443_1619 (size=17284) 2024-12-06T16:03:10,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:10,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501050764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:10,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:10,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501050770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:10,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:10,808 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,809 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,809 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:10,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:10,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:10,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501050803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:10,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501050803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:10,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:10,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:10,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501050803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:10,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:10,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:10,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:10,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501050912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:10,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:10,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:10,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501050912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:10,915 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,915 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,915 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:10,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:10,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501050912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:10,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:10,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:10,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501050912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:10,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:10,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:10,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501050923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:10,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:10,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:11,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:11,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501051122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501051122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:11,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:11,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:11,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501051122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501051122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:11,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:11,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:11,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:11,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:11,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:11,127 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:11,128 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:11,128 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:11,128 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:11,128 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:11,128 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:11,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:11,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501051132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:11,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:11,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:11,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:11,178 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:11,182 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206f11c441ade244e4c9dc1fd369d9b8a87_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206f11c441ade244e4c9dc1fd369d9b8a87_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:11,183 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/f41b6b74269b47149044ac99be1b8ab0, store: [table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:11,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/f41b6b74269b47149044ac99be1b8ab0 is 175, key is test_row_0/A:col10/1733500990106/Put/seqid=0 2024-12-06T16:03:11,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742444_1620 (size=48389) 2024-12-06T16:03:11,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:11,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501051432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:11,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:11,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501051432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:11,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:11,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:11,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:11,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:11,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501051432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:11,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:11,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:11,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:11,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:11,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501051433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:11,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:11,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:11,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:11,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:11,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:11,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:11,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-06T16:03:11,452 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:03:11,453 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:03:11,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-12-06T16:03:11,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-06T16:03:11,455 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:03:11,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:11,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501051452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:11,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:11,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:11,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:11,456 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:03:11,456 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:03:11,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-06T16:03:11,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-06T16:03:11,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:11,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:11,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:11,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:11,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:11,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:11,613 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=177, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/f41b6b74269b47149044ac99be1b8ab0 2024-12-06T16:03:11,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/45aa5cc7785e47a6a77cabc0f885d359 is 50, key is test_row_0/B:col10/1733500990106/Put/seqid=0 2024-12-06T16:03:11,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742445_1621 (size=12151) 2024-12-06T16:03:11,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-06T16:03:11,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:11,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:11,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:11,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:11,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:11,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:11,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-06T16:03:11,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-06T16:03:11,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:11,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:11,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:11,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:11,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:11,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:11,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:11,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501051942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:11,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:11,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501051942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:11,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:11,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501051942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:11,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:11,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501051942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:11,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:11,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501051962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:11,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:11,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:11,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:12,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/45aa5cc7785e47a6a77cabc0f885d359 2024-12-06T16:03:12,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/0cca58e36ce540278790e649f6c03805 is 50, key is test_row_0/C:col10/1733500990106/Put/seqid=0 2024-12-06T16:03:12,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-06T16:03:12,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:12,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:12,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:12,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:12,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:12,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:12,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-06T16:03:12,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742446_1622 (size=12151) 2024-12-06T16:03:12,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-06T16:03:12,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:12,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:12,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:12,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:12,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:12,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:12,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-06T16:03:12,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:12,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:12,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:12,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:12,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:12,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/0cca58e36ce540278790e649f6c03805 2024-12-06T16:03:12,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/f41b6b74269b47149044ac99be1b8ab0 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/f41b6b74269b47149044ac99be1b8ab0 2024-12-06T16:03:12,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/f41b6b74269b47149044ac99be1b8ab0, entries=250, sequenceid=177, filesize=47.3 K 2024-12-06T16:03:12,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/45aa5cc7785e47a6a77cabc0f885d359 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/45aa5cc7785e47a6a77cabc0f885d359 2024-12-06T16:03:12,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/45aa5cc7785e47a6a77cabc0f885d359, entries=150, sequenceid=177, filesize=11.9 K 2024-12-06T16:03:12,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-06T16:03:12,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:12,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:12,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:12,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:12,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:12,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:12,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/0cca58e36ce540278790e649f6c03805 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/0cca58e36ce540278790e649f6c03805 2024-12-06T16:03:12,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/0cca58e36ce540278790e649f6c03805, entries=150, sequenceid=177, filesize=11.9 K 2024-12-06T16:03:12,541 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for d143cebc3e2771774f1f66ab4723edf2 in 1806ms, sequenceid=177, compaction requested=true 2024-12-06T16:03:12,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:12,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d143cebc3e2771774f1f66ab4723edf2:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:03:12,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:12,541 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:03:12,541 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:03:12,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d143cebc3e2771774f1f66ab4723edf2:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:03:12,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:12,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d143cebc3e2771774f1f66ab4723edf2:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:03:12,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:03:12,542 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:03:12,542 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110964 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:03:12,542 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): d143cebc3e2771774f1f66ab4723edf2/A is initiating minor compaction (all files) 2024-12-06T16:03:12,542 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): d143cebc3e2771774f1f66ab4723edf2/B is initiating minor compaction (all files) 2024-12-06T16:03:12,542 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d143cebc3e2771774f1f66ab4723edf2/A in TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:12,542 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d143cebc3e2771774f1f66ab4723edf2/B in TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:12,542 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/eb1171617e084072b84c7260daaeb58e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/14442bf624344c27bfd2255ae5fe7cc8, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/45aa5cc7785e47a6a77cabc0f885d359] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp, totalSize=35.9 K 2024-12-06T16:03:12,542 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/e5cd51d8e5f2487cae978b7121f0c86b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/d420dae89c504f209c3cc38949e29859, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/f41b6b74269b47149044ac99be1b8ab0] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp, totalSize=108.4 K 2024-12-06T16:03:12,542 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:12,542 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. files: [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/e5cd51d8e5f2487cae978b7121f0c86b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/d420dae89c504f209c3cc38949e29859, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/f41b6b74269b47149044ac99be1b8ab0] 2024-12-06T16:03:12,543 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting e5cd51d8e5f2487cae978b7121f0c86b, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733500987118 2024-12-06T16:03:12,543 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb1171617e084072b84c7260daaeb58e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733500987118 2024-12-06T16:03:12,543 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting d420dae89c504f209c3cc38949e29859, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733500989454 2024-12-06T16:03:12,543 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14442bf624344c27bfd2255ae5fe7cc8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733500989454 2024-12-06T16:03:12,543 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting f41b6b74269b47149044ac99be1b8ab0, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733500990099 2024-12-06T16:03:12,543 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45aa5cc7785e47a6a77cabc0f885d359, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733500990106 2024-12-06T16:03:12,550 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:12,562 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d143cebc3e2771774f1f66ab4723edf2#B#compaction#531 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:03:12,562 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/964582024b7c47b593ce7e1a6d1652db is 50, key is test_row_0/B:col10/1733500990106/Put/seqid=0 2024-12-06T16:03:12,566 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120651a6ee4099024ed482d179964431a052_d143cebc3e2771774f1f66ab4723edf2 store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:12,568 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120651a6ee4099024ed482d179964431a052_d143cebc3e2771774f1f66ab4723edf2, store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:12,568 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120651a6ee4099024ed482d179964431a052_d143cebc3e2771774f1f66ab4723edf2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-06T16:03:12,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742447_1623 (size=12561) 2024-12-06T16:03:12,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742448_1624 (size=4469) 2024-12-06T16:03:12,626 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d143cebc3e2771774f1f66ab4723edf2#A#compaction#530 average throughput is 0.32 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:03:12,627 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/203e8a27442b48ed9735c54107598d42 is 175, key is test_row_0/A:col10/1733500990106/Put/seqid=0 2024-12-06T16:03:12,635 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/964582024b7c47b593ce7e1a6d1652db as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/964582024b7c47b593ce7e1a6d1652db 2024-12-06T16:03:12,643 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d143cebc3e2771774f1f66ab4723edf2/B of d143cebc3e2771774f1f66ab4723edf2 into 964582024b7c47b593ce7e1a6d1652db(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:03:12,644 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:12,644 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., storeName=d143cebc3e2771774f1f66ab4723edf2/B, priority=13, startTime=1733500992541; duration=0sec 2024-12-06T16:03:12,644 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:03:12,644 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d143cebc3e2771774f1f66ab4723edf2:B 2024-12-06T16:03:12,644 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:03:12,645 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:03:12,645 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): d143cebc3e2771774f1f66ab4723edf2/C is initiating minor compaction (all files) 2024-12-06T16:03:12,645 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d143cebc3e2771774f1f66ab4723edf2/C in TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:12,645 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/a5d3927b7ae84dac9ca3f4c8df4f2129, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/820b4be98bf74f039b00f78e56e9f1b2, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/0cca58e36ce540278790e649f6c03805] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp, totalSize=35.9 K 2024-12-06T16:03:12,646 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5d3927b7ae84dac9ca3f4c8df4f2129, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733500987118 2024-12-06T16:03:12,646 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 820b4be98bf74f039b00f78e56e9f1b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733500989454 2024-12-06T16:03:12,646 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cca58e36ce540278790e649f6c03805, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733500990106 2024-12-06T16:03:12,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742449_1625 (size=31515) 2024-12-06T16:03:12,655 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/203e8a27442b48ed9735c54107598d42 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/203e8a27442b48ed9735c54107598d42 2024-12-06T16:03:12,663 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d143cebc3e2771774f1f66ab4723edf2/A of d143cebc3e2771774f1f66ab4723edf2 into 203e8a27442b48ed9735c54107598d42(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:03:12,663 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:12,663 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., storeName=d143cebc3e2771774f1f66ab4723edf2/A, priority=13, startTime=1733500992541; duration=0sec 2024-12-06T16:03:12,663 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:12,663 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d143cebc3e2771774f1f66ab4723edf2:A 2024-12-06T16:03:12,666 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d143cebc3e2771774f1f66ab4723edf2#C#compaction#532 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:03:12,667 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/8474e40e62b843d8a883532db748eb74 is 50, key is test_row_0/C:col10/1733500990106/Put/seqid=0 2024-12-06T16:03:12,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742450_1626 (size=12561) 2024-12-06T16:03:12,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-06T16:03:12,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:12,690 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing d143cebc3e2771774f1f66ab4723edf2 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-12-06T16:03:12,703 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/8474e40e62b843d8a883532db748eb74 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/8474e40e62b843d8a883532db748eb74 2024-12-06T16:03:12,718 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d143cebc3e2771774f1f66ab4723edf2/C of d143cebc3e2771774f1f66ab4723edf2 into 8474e40e62b843d8a883532db748eb74(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:03:12,718 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:12,718 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., storeName=d143cebc3e2771774f1f66ab4723edf2/C, priority=13, startTime=1733500992541; duration=0sec 2024-12-06T16:03:12,718 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:12,718 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d143cebc3e2771774f1f66ab4723edf2:C 2024-12-06T16:03:12,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206fc2a2a8df63a4b568abec59fdb2d2980_d143cebc3e2771774f1f66ab4723edf2 is 50, key is test_row_0/A:col10/1733500990758/Put/seqid=0 2024-12-06T16:03:12,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742451_1627 (size=12304) 2024-12-06T16:03:12,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,796 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206fc2a2a8df63a4b568abec59fdb2d2980_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206fc2a2a8df63a4b568abec59fdb2d2980_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:12,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/360f4e9d90094ad99fc27bdd3aca1feb, store: [table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:12,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/360f4e9d90094ad99fc27bdd3aca1feb is 175, key is test_row_0/A:col10/1733500990758/Put/seqid=0 2024-12-06T16:03:12,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742452_1628 (size=31105) 2024-12-06T16:03:12,828 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/360f4e9d90094ad99fc27bdd3aca1feb 2024-12-06T16:03:12,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/e8616b8ef2094309a431a0b4392766eb is 50, key is test_row_0/B:col10/1733500990758/Put/seqid=0 2024-12-06T16:03:12,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742453_1629 (size=12151) 2024-12-06T16:03:12,879 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/e8616b8ef2094309a431a0b4392766eb 2024-12-06T16:03:12,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/c2497e3b1aa8461980b5ac26c28e3fd1 is 50, key is test_row_0/C:col10/1733500990758/Put/seqid=0 2024-12-06T16:03:12,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742454_1630 (size=12151) 2024-12-06T16:03:12,913 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/c2497e3b1aa8461980b5ac26c28e3fd1 2024-12-06T16:03:12,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/360f4e9d90094ad99fc27bdd3aca1feb as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/360f4e9d90094ad99fc27bdd3aca1feb 2024-12-06T16:03:12,924 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/360f4e9d90094ad99fc27bdd3aca1feb, entries=150, sequenceid=197, filesize=30.4 K 2024-12-06T16:03:12,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/e8616b8ef2094309a431a0b4392766eb as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/e8616b8ef2094309a431a0b4392766eb 2024-12-06T16:03:12,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,929 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/e8616b8ef2094309a431a0b4392766eb, entries=150, sequenceid=197, filesize=11.9 K 2024-12-06T16:03:12,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/c2497e3b1aa8461980b5ac26c28e3fd1 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/c2497e3b1aa8461980b5ac26c28e3fd1 2024-12-06T16:03:12,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,944 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/c2497e3b1aa8461980b5ac26c28e3fd1, entries=150, sequenceid=197, filesize=11.9 K 2024-12-06T16:03:12,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,945 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=0 B/0 for d143cebc3e2771774f1f66ab4723edf2 in 255ms, sequenceid=197, compaction requested=false 2024-12-06T16:03:12,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:12,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:12,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-12-06T16:03:12,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-12-06T16:03:12,949 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-06T16:03:12,949 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4910 sec 2024-12-06T16:03:12,951 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.4970 sec 2024-12-06T16:03:12,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:12,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:13,135 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:03:13,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d143cebc3e2771774f1f66ab4723edf2 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-06T16:03:13,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120632258dae56df4f1ca61b06094a3ef2ce_d143cebc3e2771774f1f66ab4723edf2 is 50, key is test_row_0/A:col10/1733500993135/Put/seqid=0 2024-12-06T16:03:13,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:13,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501053176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:13,191 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,191 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,192 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:13,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:13,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501053180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:13,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:13,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:13,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501053185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:13,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:13,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:13,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742455_1631 (size=12304) 2024-12-06T16:03:13,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501053187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:13,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:13,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501053190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:13,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:13,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:13,199 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,204 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120632258dae56df4f1ca61b06094a3ef2ce_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120632258dae56df4f1ca61b06094a3ef2ce_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:13,205 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/1cfa9013643a45e599ee57541ce0e3b2, store: [table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:13,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/1cfa9013643a45e599ee57541ce0e3b2 is 175, key is test_row_0/A:col10/1733500993135/Put/seqid=0 2024-12-06T16:03:13,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742456_1632 (size=31105) 2024-12-06T16:03:13,254 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/1cfa9013643a45e599ee57541ce0e3b2 2024-12-06T16:03:13,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/66837551ad964e4991df1d05075b2fd1 is 50, key is test_row_0/B:col10/1733500993135/Put/seqid=0 2024-12-06T16:03:13,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742457_1633 (size=12151) 2024-12-06T16:03:13,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/66837551ad964e4991df1d05075b2fd1 2024-12-06T16:03:13,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/81ceed5a49cf489387367f6b3c76fb57 is 50, key is test_row_0/C:col10/1733500993135/Put/seqid=0 2024-12-06T16:03:13,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:13,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501053302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:13,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:13,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501053302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:13,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:13,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501053302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:13,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:13,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501053303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:13,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501053303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:13,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742458_1634 (size=12151) 2024-12-06T16:03:13,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501053514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:13,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501053514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:13,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:13,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501053514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:13,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:13,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501053514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:13,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:13,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501053514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:13,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-06T16:03:13,602 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:03:13,602 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:03:13,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-06T16:03:13,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-06T16:03:13,605 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:03:13,605 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:03:13,605 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:03:13,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-06T16:03:13,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/81ceed5a49cf489387367f6b3c76fb57 2024-12-06T16:03:13,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/1cfa9013643a45e599ee57541ce0e3b2 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/1cfa9013643a45e599ee57541ce0e3b2 2024-12-06T16:03:13,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/1cfa9013643a45e599ee57541ce0e3b2, entries=150, sequenceid=211, filesize=30.4 K 2024-12-06T16:03:13,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/66837551ad964e4991df1d05075b2fd1 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/66837551ad964e4991df1d05075b2fd1 2024-12-06T16:03:13,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/66837551ad964e4991df1d05075b2fd1, entries=150, sequenceid=211, filesize=11.9 K 2024-12-06T16:03:13,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/81ceed5a49cf489387367f6b3c76fb57 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/81ceed5a49cf489387367f6b3c76fb57 2024-12-06T16:03:13,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/81ceed5a49cf489387367f6b3c76fb57, entries=150, sequenceid=211, filesize=11.9 K 2024-12-06T16:03:13,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for d143cebc3e2771774f1f66ab4723edf2 in 601ms, sequenceid=211, compaction requested=true 2024-12-06T16:03:13,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:13,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d143cebc3e2771774f1f66ab4723edf2:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:03:13,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:13,737 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:03:13,737 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:03:13,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d143cebc3e2771774f1f66ab4723edf2:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:03:13,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:13,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d143cebc3e2771774f1f66ab4723edf2:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:03:13,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:03:13,738 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:03:13,738 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93725 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:03:13,738 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): d143cebc3e2771774f1f66ab4723edf2/A is initiating minor compaction (all files) 2024-12-06T16:03:13,738 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): d143cebc3e2771774f1f66ab4723edf2/B is initiating minor compaction (all files) 2024-12-06T16:03:13,738 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d143cebc3e2771774f1f66ab4723edf2/B in TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:13,738 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d143cebc3e2771774f1f66ab4723edf2/A in TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:13,738 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/964582024b7c47b593ce7e1a6d1652db, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/e8616b8ef2094309a431a0b4392766eb, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/66837551ad964e4991df1d05075b2fd1] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp, totalSize=36.0 K 2024-12-06T16:03:13,738 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/203e8a27442b48ed9735c54107598d42, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/360f4e9d90094ad99fc27bdd3aca1feb, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/1cfa9013643a45e599ee57541ce0e3b2] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp, totalSize=91.5 K 2024-12-06T16:03:13,738 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:13,738 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. files: [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/203e8a27442b48ed9735c54107598d42, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/360f4e9d90094ad99fc27bdd3aca1feb, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/1cfa9013643a45e599ee57541ce0e3b2] 2024-12-06T16:03:13,739 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 964582024b7c47b593ce7e1a6d1652db, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733500990106 2024-12-06T16:03:13,739 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting e8616b8ef2094309a431a0b4392766eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733500990758 2024-12-06T16:03:13,740 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 66837551ad964e4991df1d05075b2fd1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733500993067 2024-12-06T16:03:13,740 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 203e8a27442b48ed9735c54107598d42, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733500990106 2024-12-06T16:03:13,740 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 360f4e9d90094ad99fc27bdd3aca1feb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733500990758 2024-12-06T16:03:13,741 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cfa9013643a45e599ee57541ce0e3b2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733500993067 2024-12-06T16:03:13,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-06T16:03:13,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:13,759 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing d143cebc3e2771774f1f66ab4723edf2 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-06T16:03:13,760 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:13,761 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d143cebc3e2771774f1f66ab4723edf2#B#compaction#539 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:03:13,762 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/56de9e083df940cdb487c9d45b94673a is 50, key is test_row_0/B:col10/1733500993135/Put/seqid=0 2024-12-06T16:03:13,764 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206b112c2f289fe4ed980d6d8db11af8167_d143cebc3e2771774f1f66ab4723edf2 store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:13,766 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206b112c2f289fe4ed980d6d8db11af8167_d143cebc3e2771774f1f66ab4723edf2, store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:13,766 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206b112c2f289fe4ed980d6d8db11af8167_d143cebc3e2771774f1f66ab4723edf2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:13,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742459_1635 (size=12663) 2024-12-06T16:03:13,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742460_1636 (size=4469) 2024-12-06T16:03:13,803 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/56de9e083df940cdb487c9d45b94673a as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/56de9e083df940cdb487c9d45b94673a 2024-12-06T16:03:13,808 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d143cebc3e2771774f1f66ab4723edf2/B of d143cebc3e2771774f1f66ab4723edf2 into 56de9e083df940cdb487c9d45b94673a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:03:13,808 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:13,808 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., storeName=d143cebc3e2771774f1f66ab4723edf2/B, priority=13, startTime=1733500993737; duration=0sec 2024-12-06T16:03:13,809 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:03:13,809 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d143cebc3e2771774f1f66ab4723edf2:B 2024-12-06T16:03:13,809 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:03:13,809 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:03:13,811 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): d143cebc3e2771774f1f66ab4723edf2/C is initiating minor compaction (all files) 2024-12-06T16:03:13,811 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d143cebc3e2771774f1f66ab4723edf2/C in TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:13,811 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/8474e40e62b843d8a883532db748eb74, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/c2497e3b1aa8461980b5ac26c28e3fd1, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/81ceed5a49cf489387367f6b3c76fb57] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp, totalSize=36.0 K 2024-12-06T16:03:13,812 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 8474e40e62b843d8a883532db748eb74, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733500990106 2024-12-06T16:03:13,812 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting c2497e3b1aa8461980b5ac26c28e3fd1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733500990758 2024-12-06T16:03:13,812 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 81ceed5a49cf489387367f6b3c76fb57, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733500993067 2024-12-06T16:03:13,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412061003bdf3bcbb43e4b492ce9a01b54727_d143cebc3e2771774f1f66ab4723edf2 is 50, key is test_row_0/A:col10/1733500993179/Put/seqid=0 2024-12-06T16:03:13,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:13,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:13,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742461_1637 (size=12304) 2024-12-06T16:03:13,836 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d143cebc3e2771774f1f66ab4723edf2#C#compaction#542 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:03:13,836 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/3fe797a0ee6a4cd19e2433ad7c303339 is 50, key is test_row_0/C:col10/1733500993135/Put/seqid=0 2024-12-06T16:03:13,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:13,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:13,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501053837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:13,844 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412061003bdf3bcbb43e4b492ce9a01b54727_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412061003bdf3bcbb43e4b492ce9a01b54727_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:13,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/279fe5be81d84caeabd2d2ffaecfd157, store: [table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:13,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/279fe5be81d84caeabd2d2ffaecfd157 is 175, key is test_row_0/A:col10/1733500993179/Put/seqid=0 2024-12-06T16:03:13,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:13,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501053839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:13,847 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,847 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,847 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:13,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:13,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501053841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:13,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:13,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501053841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:13,850 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,850 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,850 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:13,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:13,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501053842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:13,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:13,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:13,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742463_1639 (size=31105) 2024-12-06T16:03:13,864 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/279fe5be81d84caeabd2d2ffaecfd157 2024-12-06T16:03:13,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/ffd61fda1c4d4efa8875ea6ae26c6e78 is 50, key is test_row_0/B:col10/1733500993179/Put/seqid=0 2024-12-06T16:03:13,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742462_1638 (size=12663) 2024-12-06T16:03:13,920 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/3fe797a0ee6a4cd19e2433ad7c303339 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/3fe797a0ee6a4cd19e2433ad7c303339 2024-12-06T16:03:13,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-06T16:03:13,925 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d143cebc3e2771774f1f66ab4723edf2/C of d143cebc3e2771774f1f66ab4723edf2 into 3fe797a0ee6a4cd19e2433ad7c303339(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:03:13,925 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:13,925 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., storeName=d143cebc3e2771774f1f66ab4723edf2/C, priority=13, startTime=1733500993737; duration=0sec 2024-12-06T16:03:13,926 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:13,926 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d143cebc3e2771774f1f66ab4723edf2:C 2024-12-06T16:03:13,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742464_1640 (size=12151) 2024-12-06T16:03:13,937 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/ffd61fda1c4d4efa8875ea6ae26c6e78 2024-12-06T16:03:13,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/0bd70f69e94042b285cedb9a0694b2b4 is 50, key is test_row_0/C:col10/1733500993179/Put/seqid=0 2024-12-06T16:03:13,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:13,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501053952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:13,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:13,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501053952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:13,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:13,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:13,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501053952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:13,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:13,961 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,961 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,961 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:13,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:13,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501053962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:13,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:13,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501053962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:13,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:13,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:13,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:13,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742465_1641 (size=12151) 2024-12-06T16:03:13,973 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/0bd70f69e94042b285cedb9a0694b2b4 2024-12-06T16:03:13,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/279fe5be81d84caeabd2d2ffaecfd157 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/279fe5be81d84caeabd2d2ffaecfd157 2024-12-06T16:03:13,984 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/279fe5be81d84caeabd2d2ffaecfd157, entries=150, sequenceid=234, filesize=30.4 K 2024-12-06T16:03:13,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/ffd61fda1c4d4efa8875ea6ae26c6e78 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/ffd61fda1c4d4efa8875ea6ae26c6e78 2024-12-06T16:03:13,989 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/ffd61fda1c4d4efa8875ea6ae26c6e78, entries=150, sequenceid=234, filesize=11.9 K 2024-12-06T16:03:13,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/0bd70f69e94042b285cedb9a0694b2b4 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/0bd70f69e94042b285cedb9a0694b2b4 2024-12-06T16:03:13,997 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/0bd70f69e94042b285cedb9a0694b2b4, entries=150, sequenceid=234, filesize=11.9 K 2024-12-06T16:03:13,997 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for d143cebc3e2771774f1f66ab4723edf2 in 238ms, sequenceid=234, compaction requested=false 2024-12-06T16:03:13,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:13,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:13,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-06T16:03:13,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-06T16:03:14,001 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-06T16:03:14,001 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 394 msec 2024-12-06T16:03:14,003 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 399 msec 2024-12-06T16:03:14,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:14,166 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:03:14,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d143cebc3e2771774f1f66ab4723edf2 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-06T16:03:14,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206326f9b597a154edf8360d401a3b27be9_d143cebc3e2771774f1f66ab4723edf2 is 50, key is test_row_0/A:col10/1733500993839/Put/seqid=0 2024-12-06T16:03:14,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742466_1642 (size=14794) 2024-12-06T16:03:14,200 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d143cebc3e2771774f1f66ab4723edf2#A#compaction#540 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:03:14,201 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/fb4a54d3b3034049b68da506a45cd530 is 175, key is test_row_0/A:col10/1733500993135/Put/seqid=0 2024-12-06T16:03:14,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:14,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501054194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:14,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742467_1643 (size=31617) 2024-12-06T16:03:14,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:14,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:14,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501054200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:14,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501054200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:14,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501054202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:14,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501054202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:14,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:14,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-06T16:03:14,232 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:03:14,233 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:03:14,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-06T16:03:14,234 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:03:14,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-06T16:03:14,235 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:03:14,235 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:03:14,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:14,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501054312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:14,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:14,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501054312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:14,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:14,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:14,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501054312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:14,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:14,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:14,315 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501054312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:14,315 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,315 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:14,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:14,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501054312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:14,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:14,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:14,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-06T16:03:14,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-06T16:03:14,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:14,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:14,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:14,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:14,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:14,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:14,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:14,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501054522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:14,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:14,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:14,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501054522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:14,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:14,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501054522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:14,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:14,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501054523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:14,528 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,528 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,528 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:14,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:14,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501054523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:14,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:14,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:14,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:14,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-06T16:03:14,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:14,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:14,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:14,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:14,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:14,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:14,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-06T16:03:14,578 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:14,582 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206326f9b597a154edf8360d401a3b27be9_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206326f9b597a154edf8360d401a3b27be9_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:14,582 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/a24f5292c83c4f22a7b875eeadb0cb80, store: [table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:14,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/a24f5292c83c4f22a7b875eeadb0cb80 is 175, key is test_row_0/A:col10/1733500993839/Put/seqid=0 2024-12-06T16:03:14,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742468_1644 (size=39749) 2024-12-06T16:03:14,590 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/a24f5292c83c4f22a7b875eeadb0cb80 2024-12-06T16:03:14,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/5863d13f1ca64619aeacb9005e5834cd is 50, key is test_row_0/B:col10/1733500993839/Put/seqid=0 2024-12-06T16:03:14,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742469_1645 (size=12151) 2024-12-06T16:03:14,609 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/fb4a54d3b3034049b68da506a45cd530 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/fb4a54d3b3034049b68da506a45cd530 2024-12-06T16:03:14,615 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d143cebc3e2771774f1f66ab4723edf2/A of d143cebc3e2771774f1f66ab4723edf2 into fb4a54d3b3034049b68da506a45cd530(size=30.9 K), total size for store is 61.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:03:14,615 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:14,615 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., storeName=d143cebc3e2771774f1f66ab4723edf2/A, priority=13, startTime=1733500993737; duration=0sec 2024-12-06T16:03:14,615 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:14,615 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d143cebc3e2771774f1f66ab4723edf2:A 2024-12-06T16:03:14,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-06T16:03:14,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:14,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:14,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:14,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:14,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:14,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:14,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:14,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501054832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:14,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:14,836 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501054832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:14,836 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,836 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:14,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:14,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501054832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:14,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:14,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:14,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:14,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501054842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:14,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:14,846 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,846 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,846 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:14,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-06T16:03:14,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:14,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:14,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:14,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:14,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:14,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501054842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:14,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:14,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:14,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:14,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-06T16:03:15,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/5863d13f1ca64619aeacb9005e5834cd 2024-12-06T16:03:15,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-06T16:03:15,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:15,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:15,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:15,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:15,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:15,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:15,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/c617331ac0a042c6bbc641fb20fd1ebe is 50, key is test_row_0/C:col10/1733500993839/Put/seqid=0 2024-12-06T16:03:15,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742470_1646 (size=12151) 2024-12-06T16:03:15,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/c617331ac0a042c6bbc641fb20fd1ebe 2024-12-06T16:03:15,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/a24f5292c83c4f22a7b875eeadb0cb80 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/a24f5292c83c4f22a7b875eeadb0cb80 2024-12-06T16:03:15,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/a24f5292c83c4f22a7b875eeadb0cb80, entries=200, sequenceid=250, filesize=38.8 K 2024-12-06T16:03:15,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/5863d13f1ca64619aeacb9005e5834cd as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/5863d13f1ca64619aeacb9005e5834cd 2024-12-06T16:03:15,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/5863d13f1ca64619aeacb9005e5834cd, entries=150, sequenceid=250, filesize=11.9 K 2024-12-06T16:03:15,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/c617331ac0a042c6bbc641fb20fd1ebe as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/c617331ac0a042c6bbc641fb20fd1ebe 2024-12-06T16:03:15,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/c617331ac0a042c6bbc641fb20fd1ebe, entries=150, sequenceid=250, filesize=11.9 K 2024-12-06T16:03:15,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for d143cebc3e2771774f1f66ab4723edf2 in 906ms, sequenceid=250, compaction requested=true 2024-12-06T16:03:15,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:15,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d143cebc3e2771774f1f66ab4723edf2:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:03:15,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:15,073 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:03:15,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d143cebc3e2771774f1f66ab4723edf2:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:03:15,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:15,073 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:03:15,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d143cebc3e2771774f1f66ab4723edf2:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:03:15,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:03:15,074 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:03:15,074 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): d143cebc3e2771774f1f66ab4723edf2/A is initiating minor compaction (all files) 2024-12-06T16:03:15,074 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d143cebc3e2771774f1f66ab4723edf2/A in TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:15,074 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/fb4a54d3b3034049b68da506a45cd530, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/279fe5be81d84caeabd2d2ffaecfd157, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/a24f5292c83c4f22a7b875eeadb0cb80] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp, totalSize=100.1 K 2024-12-06T16:03:15,074 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:15,074 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. files: [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/fb4a54d3b3034049b68da506a45cd530, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/279fe5be81d84caeabd2d2ffaecfd157, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/a24f5292c83c4f22a7b875eeadb0cb80] 2024-12-06T16:03:15,074 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:03:15,074 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): d143cebc3e2771774f1f66ab4723edf2/B is initiating minor compaction (all files) 2024-12-06T16:03:15,074 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d143cebc3e2771774f1f66ab4723edf2/B in TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:15,074 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/56de9e083df940cdb487c9d45b94673a, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/ffd61fda1c4d4efa8875ea6ae26c6e78, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/5863d13f1ca64619aeacb9005e5834cd] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp, totalSize=36.1 K 2024-12-06T16:03:15,075 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb4a54d3b3034049b68da506a45cd530, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733500993067 2024-12-06T16:03:15,075 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 56de9e083df940cdb487c9d45b94673a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733500993067 2024-12-06T16:03:15,075 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 279fe5be81d84caeabd2d2ffaecfd157, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733500993164 2024-12-06T16:03:15,075 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting ffd61fda1c4d4efa8875ea6ae26c6e78, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733500993164 2024-12-06T16:03:15,075 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting a24f5292c83c4f22a7b875eeadb0cb80, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733500993839 2024-12-06T16:03:15,075 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 5863d13f1ca64619aeacb9005e5834cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733500993839 2024-12-06T16:03:15,098 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d143cebc3e2771774f1f66ab4723edf2#B#compaction#548 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:03:15,099 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/8487523788f44db0bdcdd125985efa6c is 50, key is test_row_0/B:col10/1733500993839/Put/seqid=0 2024-12-06T16:03:15,112 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:15,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742471_1647 (size=12765) 2024-12-06T16:03:15,141 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206926a14bd1c6b48f3aff69188d0e17b7c_d143cebc3e2771774f1f66ab4723edf2 store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:15,143 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206926a14bd1c6b48f3aff69188d0e17b7c_d143cebc3e2771774f1f66ab4723edf2, store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:15,143 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206926a14bd1c6b48f3aff69188d0e17b7c_d143cebc3e2771774f1f66ab4723edf2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:15,149 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/8487523788f44db0bdcdd125985efa6c as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/8487523788f44db0bdcdd125985efa6c 2024-12-06T16:03:15,156 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d143cebc3e2771774f1f66ab4723edf2/B of d143cebc3e2771774f1f66ab4723edf2 into 8487523788f44db0bdcdd125985efa6c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:03:15,156 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:15,156 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., storeName=d143cebc3e2771774f1f66ab4723edf2/B, priority=13, startTime=1733500995073; duration=0sec 2024-12-06T16:03:15,156 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:03:15,156 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d143cebc3e2771774f1f66ab4723edf2:B 2024-12-06T16:03:15,156 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:03:15,157 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:03:15,157 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): d143cebc3e2771774f1f66ab4723edf2/C is initiating minor compaction (all files) 2024-12-06T16:03:15,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-06T16:03:15,157 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d143cebc3e2771774f1f66ab4723edf2/C in TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:15,157 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/3fe797a0ee6a4cd19e2433ad7c303339, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/0bd70f69e94042b285cedb9a0694b2b4, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/c617331ac0a042c6bbc641fb20fd1ebe] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp, totalSize=36.1 K 2024-12-06T16:03:15,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:15,157 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing d143cebc3e2771774f1f66ab4723edf2 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-06T16:03:15,160 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fe797a0ee6a4cd19e2433ad7c303339, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733500993067 2024-12-06T16:03:15,160 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bd70f69e94042b285cedb9a0694b2b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733500993164 2024-12-06T16:03:15,160 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting c617331ac0a042c6bbc641fb20fd1ebe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733500993839 2024-12-06T16:03:15,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742472_1648 (size=4469) 2024-12-06T16:03:15,168 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d143cebc3e2771774f1f66ab4723edf2#A#compaction#549 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:03:15,170 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/6fbca934cdb94eaa86e4f737e1e4403d is 175, key is test_row_0/A:col10/1733500993839/Put/seqid=0 2024-12-06T16:03:15,186 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d143cebc3e2771774f1f66ab4723edf2#C#compaction#550 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:03:15,186 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/db8b5e15b1534591b2af62eafda6d3f2 is 50, key is test_row_0/C:col10/1733500993839/Put/seqid=0 2024-12-06T16:03:15,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206ed090a1656b04de6b7e0ae6b980cf6c2_d143cebc3e2771774f1f66ab4723edf2 is 50, key is test_row_0/A:col10/1733500994200/Put/seqid=0 2024-12-06T16:03:15,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742473_1649 (size=31719) 2024-12-06T16:03:15,207 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/6fbca934cdb94eaa86e4f737e1e4403d as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/6fbca934cdb94eaa86e4f737e1e4403d 2024-12-06T16:03:15,212 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d143cebc3e2771774f1f66ab4723edf2/A of d143cebc3e2771774f1f66ab4723edf2 into 6fbca934cdb94eaa86e4f737e1e4403d(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:03:15,212 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:15,212 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., storeName=d143cebc3e2771774f1f66ab4723edf2/A, priority=13, startTime=1733500995073; duration=0sec 2024-12-06T16:03:15,212 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:15,212 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d143cebc3e2771774f1f66ab4723edf2:A 2024-12-06T16:03:15,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742475_1651 (size=12454) 2024-12-06T16:03:15,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742474_1650 (size=12765) 2024-12-06T16:03:15,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:15,227 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206ed090a1656b04de6b7e0ae6b980cf6c2_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ed090a1656b04de6b7e0ae6b980cf6c2_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:15,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/e9344b29af88435597450c097523b075, store: [table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:15,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/e9344b29af88435597450c097523b075 is 175, key is test_row_0/A:col10/1733500994200/Put/seqid=0 2024-12-06T16:03:15,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742476_1652 (size=31255) 2024-12-06T16:03:15,241 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/e9344b29af88435597450c097523b075 2024-12-06T16:03:15,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/67b912624ff84ea2b1e72476056b9628 is 50, key is test_row_0/B:col10/1733500994200/Put/seqid=0 2024-12-06T16:03:15,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742477_1653 (size=12301) 2024-12-06T16:03:15,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:15,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-06T16:03:15,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:15,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501055362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:15,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:15,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:15,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:15,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:15,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501055365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:15,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:15,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501055365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:15,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:15,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:15,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501055372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:15,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501055372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:15,380 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:15,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:15,380 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:15,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:15,380 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:15,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:15,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:15,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501055482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:15,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:15,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:15,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:15,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:15,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501055482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:15,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:15,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:15,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501055482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:15,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:15,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:15,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:15,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:15,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501055482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:15,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:15,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:15,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:15,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:15,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:15,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:15,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501055482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:15,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:15,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:15,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:15,623 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/db8b5e15b1534591b2af62eafda6d3f2 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/db8b5e15b1534591b2af62eafda6d3f2 2024-12-06T16:03:15,629 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d143cebc3e2771774f1f66ab4723edf2/C of d143cebc3e2771774f1f66ab4723edf2 into db8b5e15b1534591b2af62eafda6d3f2(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:03:15,629 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:15,629 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., storeName=d143cebc3e2771774f1f66ab4723edf2/C, priority=13, startTime=1733500995073; duration=0sec 2024-12-06T16:03:15,629 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:15,629 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d143cebc3e2771774f1f66ab4723edf2:C 2024-12-06T16:03:15,693 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/67b912624ff84ea2b1e72476056b9628 2024-12-06T16:03:15,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:15,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501055692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:15,696 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:15,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:15,696 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:15,696 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:15,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501055692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:15,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:15,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:15,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:15,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:15,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501055692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:15,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:15,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:15,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:15,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501055693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:15,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:15,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:15,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501055693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:15,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:15,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:15,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:15,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:15,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:15,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:15,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/2672d0e222c94944b7484b88768a7e85 is 50, key is test_row_0/C:col10/1733500994200/Put/seqid=0 2024-12-06T16:03:15,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742478_1654 (size=12301) 2024-12-06T16:03:16,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:16,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:16,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501056003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:16,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501056003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:16,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:16,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:16,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501056003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:16,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501056003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:16,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:16,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501056003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:16,114 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/2672d0e222c94944b7484b88768a7e85 2024-12-06T16:03:16,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/e9344b29af88435597450c097523b075 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/e9344b29af88435597450c097523b075 2024-12-06T16:03:16,124 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/e9344b29af88435597450c097523b075, entries=150, sequenceid=275, filesize=30.5 K 2024-12-06T16:03:16,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/67b912624ff84ea2b1e72476056b9628 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/67b912624ff84ea2b1e72476056b9628 2024-12-06T16:03:16,128 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/67b912624ff84ea2b1e72476056b9628, entries=150, sequenceid=275, filesize=12.0 K 2024-12-06T16:03:16,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/2672d0e222c94944b7484b88768a7e85 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/2672d0e222c94944b7484b88768a7e85 2024-12-06T16:03:16,144 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/2672d0e222c94944b7484b88768a7e85, entries=150, sequenceid=275, filesize=12.0 K 2024-12-06T16:03:16,145 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for d143cebc3e2771774f1f66ab4723edf2 in 988ms, sequenceid=275, compaction requested=false 2024-12-06T16:03:16,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:16,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:16,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-06T16:03:16,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-06T16:03:16,151 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-06T16:03:16,151 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9120 sec 2024-12-06T16:03:16,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.9190 sec 2024-12-06T16:03:16,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-06T16:03:16,383 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:03:16,383 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:03:16,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-06T16:03:16,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-06T16:03:16,386 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:03:16,387 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:03:16,387 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:03:16,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-06T16:03:16,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:16,515 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:03:16,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d143cebc3e2771774f1f66ab4723edf2 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-06T16:03:16,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206caf8516b9ec9411a9711bbdd11491267_d143cebc3e2771774f1f66ab4723edf2 is 50, key is test_row_0/A:col10/1733500995353/Put/seqid=0 2024-12-06T16:03:16,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-06T16:03:16,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:16,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:16,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:16,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:16,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:16,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:16,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:16,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501056548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:16,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,558 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,558 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:16,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742479_1655 (size=12454) 2024-12-06T16:03:16,562 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:16,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:16,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501056553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:16,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:16,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:16,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501056556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:16,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:16,566 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206caf8516b9ec9411a9711bbdd11491267_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206caf8516b9ec9411a9711bbdd11491267_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:16,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501056556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:16,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:16,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:16,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501056557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:16,567 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/e64b41924b7b44a2b3a4606c4cdf2ef2, store: [table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:16,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/e64b41924b7b44a2b3a4606c4cdf2ef2 is 175, key is test_row_0/A:col10/1733500995353/Put/seqid=0 2024-12-06T16:03:16,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:16,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:16,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742480_1656 (size=31255) 2024-12-06T16:03:16,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:16,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501056663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:16,666 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,666 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,666 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:16,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:16,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501056672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:16,679 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:16,679 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:16,679 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:16,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501056672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:16,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501056672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:16,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:16,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:16,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501056672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:16,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:16,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:16,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-06T16:03:16,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:16,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:16,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:16,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:16,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:16,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:16,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-06T16:03:16,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-06T16:03:16,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:16,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:16,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:16,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:16,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:16,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:16,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:16,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501056871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:16,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:16,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:16,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501056882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:16,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:16,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:16,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501056891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:16,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:16,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:16,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501056892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:16,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501056892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:16,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:16,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:16,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:16,994 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/e64b41924b7b44a2b3a4606c4cdf2ef2 2024-12-06T16:03:17,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-06T16:03:17,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:17,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:17,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:17,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:17,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:17,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:17,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/a942aebedecc43a599c226a14f480add is 50, key is test_row_0/B:col10/1733500995353/Put/seqid=0 2024-12-06T16:03:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-06T16:03:17,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742481_1657 (size=12301) 2024-12-06T16:03:17,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/a942aebedecc43a599c226a14f480add 2024-12-06T16:03:17,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/23bf097b587a4723b05d5b0ce1b72204 is 50, key is test_row_0/C:col10/1733500995353/Put/seqid=0 2024-12-06T16:03:17,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742482_1658 (size=12301) 2024-12-06T16:03:17,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-06T16:03:17,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:17,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:17,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:17,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:17,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:17,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:17,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:17,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501057182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:17,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:17,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501057192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:17,196 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:17,196 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:17,196 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:17,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:17,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501057201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:17,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:17,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501057202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:17,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:17,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501057202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:17,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:17,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:17,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:17,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:17,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:17,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:17,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:17,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:17,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:17,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-06T16:03:17,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:17,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:17,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:17,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:17,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:17,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:17,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-06T16:03:17,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:17,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:17,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:17,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:17,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:17,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:17,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/23bf097b587a4723b05d5b0ce1b72204 2024-12-06T16:03:17,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/e64b41924b7b44a2b3a4606c4cdf2ef2 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/e64b41924b7b44a2b3a4606c4cdf2ef2 2024-12-06T16:03:17,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/e64b41924b7b44a2b3a4606c4cdf2ef2, entries=150, sequenceid=291, filesize=30.5 K 2024-12-06T16:03:17,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/a942aebedecc43a599c226a14f480add as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/a942aebedecc43a599c226a14f480add 2024-12-06T16:03:17,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/a942aebedecc43a599c226a14f480add, entries=150, sequenceid=291, filesize=12.0 K 2024-12-06T16:03:17,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/23bf097b587a4723b05d5b0ce1b72204 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/23bf097b587a4723b05d5b0ce1b72204 2024-12-06T16:03:17,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/23bf097b587a4723b05d5b0ce1b72204, entries=150, sequenceid=291, filesize=12.0 K 2024-12-06T16:03:17,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for d143cebc3e2771774f1f66ab4723edf2 in 988ms, sequenceid=291, compaction requested=true 2024-12-06T16:03:17,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:17,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d143cebc3e2771774f1f66ab4723edf2:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:03:17,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:17,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d143cebc3e2771774f1f66ab4723edf2:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:03:17,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:17,504 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:03:17,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d143cebc3e2771774f1f66ab4723edf2:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:03:17,504 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:03:17,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:03:17,505 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:03:17,505 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:03:17,505 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): d143cebc3e2771774f1f66ab4723edf2/A is initiating minor compaction (all files) 2024-12-06T16:03:17,505 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): d143cebc3e2771774f1f66ab4723edf2/B is initiating minor compaction (all files) 2024-12-06T16:03:17,505 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d143cebc3e2771774f1f66ab4723edf2/B in TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:17,505 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d143cebc3e2771774f1f66ab4723edf2/A in TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:17,505 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/8487523788f44db0bdcdd125985efa6c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/67b912624ff84ea2b1e72476056b9628, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/a942aebedecc43a599c226a14f480add] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp, totalSize=36.5 K 2024-12-06T16:03:17,505 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/6fbca934cdb94eaa86e4f737e1e4403d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/e9344b29af88435597450c097523b075, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/e64b41924b7b44a2b3a4606c4cdf2ef2] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp, totalSize=92.0 K 2024-12-06T16:03:17,505 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:17,505 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. files: [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/6fbca934cdb94eaa86e4f737e1e4403d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/e9344b29af88435597450c097523b075, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/e64b41924b7b44a2b3a4606c4cdf2ef2] 2024-12-06T16:03:17,505 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 8487523788f44db0bdcdd125985efa6c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733500993839 2024-12-06T16:03:17,505 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fbca934cdb94eaa86e4f737e1e4403d, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733500993839 2024-12-06T16:03:17,506 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 67b912624ff84ea2b1e72476056b9628, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733500994190 2024-12-06T16:03:17,506 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9344b29af88435597450c097523b075, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733500994190 2024-12-06T16:03:17,506 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting a942aebedecc43a599c226a14f480add, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733500995353 2024-12-06T16:03:17,506 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting e64b41924b7b44a2b3a4606c4cdf2ef2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733500995353 2024-12-06T16:03:17,512 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:17,513 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d143cebc3e2771774f1f66ab4723edf2#B#compaction#557 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:03:17,513 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/66af06dd8b1641b6a30ccbf83b5525fa is 50, key is test_row_0/B:col10/1733500995353/Put/seqid=0 2024-12-06T16:03:17,515 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412068b516bae648449d59232e095569ed78c_d143cebc3e2771774f1f66ab4723edf2 store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:17,517 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412068b516bae648449d59232e095569ed78c_d143cebc3e2771774f1f66ab4723edf2, store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:17,517 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412068b516bae648449d59232e095569ed78c_d143cebc3e2771774f1f66ab4723edf2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:17,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-06T16:03:17,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742483_1659 (size=13017) 2024-12-06T16:03:17,540 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/66af06dd8b1641b6a30ccbf83b5525fa as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/66af06dd8b1641b6a30ccbf83b5525fa 2024-12-06T16:03:17,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742484_1660 (size=4469) 2024-12-06T16:03:17,548 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d143cebc3e2771774f1f66ab4723edf2/B of d143cebc3e2771774f1f66ab4723edf2 into 66af06dd8b1641b6a30ccbf83b5525fa(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:03:17,548 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:17,548 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., storeName=d143cebc3e2771774f1f66ab4723edf2/B, priority=13, startTime=1733500997504; duration=0sec 2024-12-06T16:03:17,548 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:03:17,548 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d143cebc3e2771774f1f66ab4723edf2:B 2024-12-06T16:03:17,548 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:03:17,549 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:03:17,549 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): d143cebc3e2771774f1f66ab4723edf2/C is initiating minor compaction (all files) 2024-12-06T16:03:17,549 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d143cebc3e2771774f1f66ab4723edf2/C in TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:17,550 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/db8b5e15b1534591b2af62eafda6d3f2, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/2672d0e222c94944b7484b88768a7e85, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/23bf097b587a4723b05d5b0ce1b72204] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp, totalSize=36.5 K 2024-12-06T16:03:17,550 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting db8b5e15b1534591b2af62eafda6d3f2, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733500993839 2024-12-06T16:03:17,551 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 2672d0e222c94944b7484b88768a7e85, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733500994190 2024-12-06T16:03:17,551 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d143cebc3e2771774f1f66ab4723edf2#A#compaction#558 average throughput is 0.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:03:17,552 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/7b3fa049fe5842569ea766049cf60c4e is 175, key is test_row_0/A:col10/1733500995353/Put/seqid=0 2024-12-06T16:03:17,552 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 23bf097b587a4723b05d5b0ce1b72204, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733500995353 2024-12-06T16:03:17,569 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d143cebc3e2771774f1f66ab4723edf2#C#compaction#559 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:03:17,570 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/9b7f54c3d6f943f5b2b48ef78aa13d58 is 50, key is test_row_0/C:col10/1733500995353/Put/seqid=0 2024-12-06T16:03:17,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742485_1661 (size=31971) 2024-12-06T16:03:17,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742486_1662 (size=13017) 2024-12-06T16:03:17,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-06T16:03:17,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:17,619 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing d143cebc3e2771774f1f66ab4723edf2 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-06T16:03:17,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206a234dbd68de444e5ac295595cca12061_d143cebc3e2771774f1f66ab4723edf2 is 50, key is test_row_0/A:col10/1733500996554/Put/seqid=0 2024-12-06T16:03:17,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742487_1663 (size=12454) 2024-12-06T16:03:17,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:17,638 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206a234dbd68de444e5ac295595cca12061_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a234dbd68de444e5ac295595cca12061_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:17,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/84deb76d45a1440b8213af9c2a9b3239, store: [table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:17,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/84deb76d45a1440b8213af9c2a9b3239 is 175, key is test_row_0/A:col10/1733500996554/Put/seqid=0 2024-12-06T16:03:17,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742488_1664 (size=31255) 2024-12-06T16:03:17,652 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=314, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/84deb76d45a1440b8213af9c2a9b3239 2024-12-06T16:03:17,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/b0e658bd3f6c4749abe25937190f6021 is 50, key is test_row_0/B:col10/1733500996554/Put/seqid=0 2024-12-06T16:03:17,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742489_1665 (size=12301) 2024-12-06T16:03:17,692 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/b0e658bd3f6c4749abe25937190f6021 2024-12-06T16:03:17,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:17,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:17,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/8b415e7a6778462288cd27773fcf0b35 is 50, key is test_row_0/C:col10/1733500996554/Put/seqid=0 2024-12-06T16:03:17,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742490_1666 (size=12301) 2024-12-06T16:03:17,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:17,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501057717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:17,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:17,726 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:17,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501057717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:17,726 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:17,726 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:17,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:17,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:17,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:17,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501057718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:17,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:17,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:17,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:17,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:17,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:17,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:17,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501057718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:17,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501057719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:17,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:17,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:17,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:17,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:17,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:17,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:17,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:17,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501057832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:17,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:17,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:17,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:17,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:17,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501057832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:17,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:17,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501057832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:17,835 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:17,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:17,835 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:17,835 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:17,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:17,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501057832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:17,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501057833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:17,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:17,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:17,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:17,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:17,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:17,836 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:17,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:17,836 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:17,836 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:17,979 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/7b3fa049fe5842569ea766049cf60c4e as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/7b3fa049fe5842569ea766049cf60c4e 2024-12-06T16:03:17,986 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d143cebc3e2771774f1f66ab4723edf2/A of d143cebc3e2771774f1f66ab4723edf2 into 7b3fa049fe5842569ea766049cf60c4e(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:03:17,986 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:17,986 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., storeName=d143cebc3e2771774f1f66ab4723edf2/A, priority=13, startTime=1733500997504; duration=0sec 2024-12-06T16:03:17,986 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:17,986 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d143cebc3e2771774f1f66ab4723edf2:A 2024-12-06T16:03:17,998 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/9b7f54c3d6f943f5b2b48ef78aa13d58 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/9b7f54c3d6f943f5b2b48ef78aa13d58 2024-12-06T16:03:18,003 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d143cebc3e2771774f1f66ab4723edf2/C of d143cebc3e2771774f1f66ab4723edf2 into 9b7f54c3d6f943f5b2b48ef78aa13d58(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:03:18,003 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:18,003 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., storeName=d143cebc3e2771774f1f66ab4723edf2/C, priority=13, startTime=1733500997504; duration=0sec 2024-12-06T16:03:18,003 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:18,003 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d143cebc3e2771774f1f66ab4723edf2:C 2024-12-06T16:03:18,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:18,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501058042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:18,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:18,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:18,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501058043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:18,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:18,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501058043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:18,047 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,047 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,047 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:18,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:18,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501058043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:18,047 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,047 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,047 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:18,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:18,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:18,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501058043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:18,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:18,111 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/8b415e7a6778462288cd27773fcf0b35 2024-12-06T16:03:18,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/84deb76d45a1440b8213af9c2a9b3239 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/84deb76d45a1440b8213af9c2a9b3239 2024-12-06T16:03:18,120 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/84deb76d45a1440b8213af9c2a9b3239, entries=150, sequenceid=314, filesize=30.5 K 2024-12-06T16:03:18,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/b0e658bd3f6c4749abe25937190f6021 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/b0e658bd3f6c4749abe25937190f6021 2024-12-06T16:03:18,125 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/b0e658bd3f6c4749abe25937190f6021, entries=150, sequenceid=314, filesize=12.0 K 2024-12-06T16:03:18,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/8b415e7a6778462288cd27773fcf0b35 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/8b415e7a6778462288cd27773fcf0b35 2024-12-06T16:03:18,133 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/8b415e7a6778462288cd27773fcf0b35, entries=150, sequenceid=314, filesize=12.0 K 2024-12-06T16:03:18,135 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for d143cebc3e2771774f1f66ab4723edf2 in 517ms, sequenceid=314, compaction requested=false 2024-12-06T16:03:18,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:18,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:18,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-06T16:03:18,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-06T16:03:18,140 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-06T16:03:18,140 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7510 sec 2024-12-06T16:03:18,142 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.7580 sec 2024-12-06T16:03:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:18,356 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:03:18,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d143cebc3e2771774f1f66ab4723edf2 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-06T16:03:18,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206061b740f2fd74a0db76dba68a4645daf_d143cebc3e2771774f1f66ab4723edf2 is 50, key is test_row_0/A:col10/1733500998355/Put/seqid=0 2024-12-06T16:03:18,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:18,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501058381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:18,389 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,389 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:18,389 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:18,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501058384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:18,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:18,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501058386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:18,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:18,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501058387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:18,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:18,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:18,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501058389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:18,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,397 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:18,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742491_1667 (size=12454) 2024-12-06T16:03:18,407 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:18,411 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206061b740f2fd74a0db76dba68a4645daf_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206061b740f2fd74a0db76dba68a4645daf_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:18,412 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/8f5f3b340f8a4ef69cb74085ed83928b, store: [table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:18,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/8f5f3b340f8a4ef69cb74085ed83928b is 175, key is test_row_0/A:col10/1733500998355/Put/seqid=0 2024-12-06T16:03:18,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742492_1668 (size=31255) 2024-12-06T16:03:18,446 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=332, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/8f5f3b340f8a4ef69cb74085ed83928b 2024-12-06T16:03:18,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/1a2e038c60d6481aa6054c2d741d6e78 is 50, key is test_row_0/B:col10/1733500998355/Put/seqid=0 2024-12-06T16:03:18,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742493_1669 (size=12301) 2024-12-06T16:03:18,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:18,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501058492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:18,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:18,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:18,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501058493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:18,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/1a2e038c60d6481aa6054c2d741d6e78 2024-12-06T16:03:18,499 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,499 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,499 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:18,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:18,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501058502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:18,504 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,504 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,504 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:18,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:18,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501058508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:18,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:18,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501058508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:18,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:18,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/d898dc57982347e1a7b6610081312c62 is 50, key is test_row_0/C:col10/1733500998355/Put/seqid=0 2024-12-06T16:03:18,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:18,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742494_1670 (size=12301) 2024-12-06T16:03:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-06T16:03:18,532 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:03:18,533 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:03:18,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-06T16:03:18,535 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:03:18,536 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:03:18,536 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:03:18,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-06T16:03:18,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-06T16:03:18,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-06T16:03:18,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:18,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:18,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:18,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:18,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:18,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:18,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:18,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501058702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:18,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:18,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501058702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:18,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:18,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:18,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501058712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:18,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501058712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:18,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:18,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:18,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501058721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:18,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:18,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:18,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-06T16:03:18,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-06T16:03:18,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:18,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:18,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:18,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:18,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:18,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/d898dc57982347e1a7b6610081312c62 2024-12-06T16:03:18,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/8f5f3b340f8a4ef69cb74085ed83928b as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/8f5f3b340f8a4ef69cb74085ed83928b 2024-12-06T16:03:18,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/8f5f3b340f8a4ef69cb74085ed83928b, entries=150, sequenceid=332, filesize=30.5 K 2024-12-06T16:03:18,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/1a2e038c60d6481aa6054c2d741d6e78 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/1a2e038c60d6481aa6054c2d741d6e78 2024-12-06T16:03:18,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/1a2e038c60d6481aa6054c2d741d6e78, entries=150, sequenceid=332, filesize=12.0 K 2024-12-06T16:03:18,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/d898dc57982347e1a7b6610081312c62 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/d898dc57982347e1a7b6610081312c62 2024-12-06T16:03:18,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/d898dc57982347e1a7b6610081312c62, entries=150, sequenceid=332, filesize=12.0 K 2024-12-06T16:03:18,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for d143cebc3e2771774f1f66ab4723edf2 in 580ms, sequenceid=332, compaction requested=true 2024-12-06T16:03:18,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:18,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d143cebc3e2771774f1f66ab4723edf2:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:03:18,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:18,936 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:03:18,936 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:03:18,937 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:03:18,937 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): d143cebc3e2771774f1f66ab4723edf2/A is initiating minor compaction (all files) 2024-12-06T16:03:18,937 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d143cebc3e2771774f1f66ab4723edf2/A in TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:18,937 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/7b3fa049fe5842569ea766049cf60c4e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/84deb76d45a1440b8213af9c2a9b3239, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/8f5f3b340f8a4ef69cb74085ed83928b] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp, totalSize=92.3 K 2024-12-06T16:03:18,937 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:18,937 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. files: [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/7b3fa049fe5842569ea766049cf60c4e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/84deb76d45a1440b8213af9c2a9b3239, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/8f5f3b340f8a4ef69cb74085ed83928b] 2024-12-06T16:03:18,937 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b3fa049fe5842569ea766049cf60c4e, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733500995353 2024-12-06T16:03:18,938 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:03:18,938 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): d143cebc3e2771774f1f66ab4723edf2/B is initiating minor compaction (all files) 2024-12-06T16:03:18,938 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d143cebc3e2771774f1f66ab4723edf2/B in TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:18,938 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/66af06dd8b1641b6a30ccbf83b5525fa, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/b0e658bd3f6c4749abe25937190f6021, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/1a2e038c60d6481aa6054c2d741d6e78] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp, totalSize=36.7 K 2024-12-06T16:03:18,938 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84deb76d45a1440b8213af9c2a9b3239, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733500996553 2024-12-06T16:03:18,938 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 66af06dd8b1641b6a30ccbf83b5525fa, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733500995353 2024-12-06T16:03:18,938 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f5f3b340f8a4ef69cb74085ed83928b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733500997717 2024-12-06T16:03:18,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d143cebc3e2771774f1f66ab4723edf2:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:03:18,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:18,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d143cebc3e2771774f1f66ab4723edf2:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:03:18,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:03:18,939 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting b0e658bd3f6c4749abe25937190f6021, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733500996553 2024-12-06T16:03:18,939 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a2e038c60d6481aa6054c2d741d6e78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733500997717 2024-12-06T16:03:18,947 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:18,949 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d143cebc3e2771774f1f66ab4723edf2#B#compaction#567 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:03:18,950 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/60b619c45d5a4336878ed0a0dc96e5e4 is 50, key is test_row_0/B:col10/1733500998355/Put/seqid=0 2024-12-06T16:03:18,959 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120617376a93bdf3466abaa77c88c564f031_d143cebc3e2771774f1f66ab4723edf2 store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:18,961 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120617376a93bdf3466abaa77c88c564f031_d143cebc3e2771774f1f66ab4723edf2, store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:18,961 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120617376a93bdf3466abaa77c88c564f031_d143cebc3e2771774f1f66ab4723edf2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:18,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742495_1671 (size=13119) 2024-12-06T16:03:18,992 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/60b619c45d5a4336878ed0a0dc96e5e4 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/60b619c45d5a4336878ed0a0dc96e5e4 2024-12-06T16:03:19,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-06T16:03:19,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:19,005 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing d143cebc3e2771774f1f66ab4723edf2 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-06T16:03:19,006 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d143cebc3e2771774f1f66ab4723edf2/B of d143cebc3e2771774f1f66ab4723edf2 into 60b619c45d5a4336878ed0a0dc96e5e4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:03:19,006 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:19,006 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., storeName=d143cebc3e2771774f1f66ab4723edf2/B, priority=13, startTime=1733500998936; duration=0sec 2024-12-06T16:03:19,006 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:03:19,006 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d143cebc3e2771774f1f66ab4723edf2:B 2024-12-06T16:03:19,006 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:03:19,009 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:03:19,009 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): d143cebc3e2771774f1f66ab4723edf2/C is initiating minor compaction (all files) 2024-12-06T16:03:19,009 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d143cebc3e2771774f1f66ab4723edf2/C in TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:19,009 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/9b7f54c3d6f943f5b2b48ef78aa13d58, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/8b415e7a6778462288cd27773fcf0b35, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/d898dc57982347e1a7b6610081312c62] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp, totalSize=36.7 K 2024-12-06T16:03:19,009 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b7f54c3d6f943f5b2b48ef78aa13d58, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733500995353 2024-12-06T16:03:19,010 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b415e7a6778462288cd27773fcf0b35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733500996553 2024-12-06T16:03:19,010 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting d898dc57982347e1a7b6610081312c62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733500997717 2024-12-06T16:03:19,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742496_1672 (size=4469) 2024-12-06T16:03:19,015 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d143cebc3e2771774f1f66ab4723edf2#A#compaction#566 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:03:19,016 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/28b65d67c21f496c9eff4eccc1bd2154 is 175, key is test_row_0/A:col10/1733500998355/Put/seqid=0 2024-12-06T16:03:19,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:19,030 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d143cebc3e2771774f1f66ab4723edf2#C#compaction#568 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:03:19,031 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/811fad4776394d999817203763360d8f is 50, key is test_row_0/C:col10/1733500998355/Put/seqid=0 2024-12-06T16:03:19,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412065fe550e45a2842e0bd66362a7c505a38_d143cebc3e2771774f1f66ab4723edf2 is 50, key is test_row_0/A:col10/1733500998386/Put/seqid=0 2024-12-06T16:03:19,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742497_1673 (size=32073) 2024-12-06T16:03:19,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501059038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:19,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:19,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501059041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:19,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:19,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501059041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:19,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:19,049 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,049 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,049 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501059042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:19,049 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/28b65d67c21f496c9eff4eccc1bd2154 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/28b65d67c21f496c9eff4eccc1bd2154 2024-12-06T16:03:19,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:19,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501059048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:19,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:19,063 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d143cebc3e2771774f1f66ab4723edf2/A of d143cebc3e2771774f1f66ab4723edf2 into 28b65d67c21f496c9eff4eccc1bd2154(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:03:19,063 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:19,063 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., storeName=d143cebc3e2771774f1f66ab4723edf2/A, priority=13, startTime=1733500998936; duration=0sec 2024-12-06T16:03:19,063 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:19,063 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d143cebc3e2771774f1f66ab4723edf2:A 2024-12-06T16:03:19,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742498_1674 (size=13119) 2024-12-06T16:03:19,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742499_1675 (size=12454) 2024-12-06T16:03:19,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:19,080 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412065fe550e45a2842e0bd66362a7c505a38_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412065fe550e45a2842e0bd66362a7c505a38_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:19,081 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/811fad4776394d999817203763360d8f as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/811fad4776394d999817203763360d8f 2024-12-06T16:03:19,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/4b35fe7b83654882b9627b6c49d0535c, store: [table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:19,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/4b35fe7b83654882b9627b6c49d0535c is 175, key is test_row_0/A:col10/1733500998386/Put/seqid=0 2024-12-06T16:03:19,088 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d143cebc3e2771774f1f66ab4723edf2/C of d143cebc3e2771774f1f66ab4723edf2 into 811fad4776394d999817203763360d8f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:03:19,089 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:19,089 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., storeName=d143cebc3e2771774f1f66ab4723edf2/C, priority=13, startTime=1733500998938; duration=0sec 2024-12-06T16:03:19,089 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:19,089 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d143cebc3e2771774f1f66ab4723edf2:C 2024-12-06T16:03:19,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742500_1676 (size=31255) 2024-12-06T16:03:19,096 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=354, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/4b35fe7b83654882b9627b6c49d0535c 2024-12-06T16:03:19,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/815f1befa7874017b38d70238c6bb7d6 is 50, key is test_row_0/B:col10/1733500998386/Put/seqid=0 2024-12-06T16:03:19,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742501_1677 (size=12301) 2024-12-06T16:03:19,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501059152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:19,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501059153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:19,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501059153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:19,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501059153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:19,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:19,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-06T16:03:19,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:19,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501059162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:19,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:19,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:19,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501059362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:19,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:19,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501059362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:19,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:19,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:19,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501059362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:19,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501059362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:19,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:19,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501059371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:19,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:19,515 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/815f1befa7874017b38d70238c6bb7d6 2024-12-06T16:03:19,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/a4a7fe1386024ac6b3984ca83d53e8e7 is 50, key is test_row_0/C:col10/1733500998386/Put/seqid=0 2024-12-06T16:03:19,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742502_1678 (size=12301) 2024-12-06T16:03:19,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-06T16:03:19,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:19,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501059672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:19,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:19,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:19,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501059672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:19,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:19,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:19,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501059672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:19,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:19,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501059672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:19,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:19,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:19,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:19,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501059681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:19,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:19,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:19,926 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/a4a7fe1386024ac6b3984ca83d53e8e7 2024-12-06T16:03:19,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/4b35fe7b83654882b9627b6c49d0535c as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/4b35fe7b83654882b9627b6c49d0535c 2024-12-06T16:03:19,936 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/4b35fe7b83654882b9627b6c49d0535c, entries=150, sequenceid=354, filesize=30.5 K 2024-12-06T16:03:19,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/815f1befa7874017b38d70238c6bb7d6 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/815f1befa7874017b38d70238c6bb7d6 2024-12-06T16:03:19,942 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/815f1befa7874017b38d70238c6bb7d6, entries=150, sequenceid=354, filesize=12.0 K 2024-12-06T16:03:19,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/a4a7fe1386024ac6b3984ca83d53e8e7 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/a4a7fe1386024ac6b3984ca83d53e8e7 2024-12-06T16:03:19,949 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/a4a7fe1386024ac6b3984ca83d53e8e7, entries=150, sequenceid=354, filesize=12.0 K 2024-12-06T16:03:19,950 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for d143cebc3e2771774f1f66ab4723edf2 in 945ms, sequenceid=354, compaction requested=false 2024-12-06T16:03:19,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:19,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:19,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-06T16:03:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-06T16:03:19,958 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-06T16:03:19,958 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4160 sec 2024-12-06T16:03:19,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.4250 sec 2024-12-06T16:03:20,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:20,196 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-06T16:03:20,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d143cebc3e2771774f1f66ab4723edf2 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-12-06T16:03:20,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206aa7c17f61a4941b3b7fe15deb35aa4e3_d143cebc3e2771774f1f66ab4723edf2 is 50, key is test_row_0/A:col10/1733501000190/Put/seqid=0 2024-12-06T16:03:20,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:20,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501060216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:20,222 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,222 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,222 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:20,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:20,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501060218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:20,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:20,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:20,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501060218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:20,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501060220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:20,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:20,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:20,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501060222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:20,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:20,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:20,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742503_1679 (size=14994) 2024-12-06T16:03:20,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:20,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501060333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:20,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501060333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:20,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501060334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:20,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:20,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501060342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:20,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:20,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501060342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:20,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:20,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:20,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501060541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:20,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:20,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:20,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501060542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:20,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:20,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:20,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501060542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:20,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:20,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:20,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501060553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:20,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:20,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:20,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501060562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:20,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:20,660 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:20,664 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206aa7c17f61a4941b3b7fe15deb35aa4e3_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206aa7c17f61a4941b3b7fe15deb35aa4e3_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:20,665 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/3cfe12b939cb4100a0fff7b397935612, store: [table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:20,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/3cfe12b939cb4100a0fff7b397935612 is 175, key is test_row_0/A:col10/1733501000190/Put/seqid=0 2024-12-06T16:03:20,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-06T16:03:20,682 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:03:20,684 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-06T16:03:20,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-06T16:03:20,687 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-06T16:03:20,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-06T16:03:20,688 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-06T16:03:20,688 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-06T16:03:20,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742504_1680 (size=39949) 2024-12-06T16:03:20,695 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=375, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/3cfe12b939cb4100a0fff7b397935612 2024-12-06T16:03:20,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/eb9965b9306e4efe802651895037f4ea is 50, key is test_row_0/B:col10/1733501000190/Put/seqid=0 2024-12-06T16:03:20,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742505_1681 (size=12301) 2024-12-06T16:03:20,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-06T16:03:20,829 INFO [Thread-2553 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2553. 2024-12-06T16:03:20,829 DEBUG [Thread-2553 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:03:20,829 DEBUG [Thread-2553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:20,829 DEBUG [Thread-2553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:20,829 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:03:20,831 INFO [Thread-2555 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2555. 2024-12-06T16:03:20,831 DEBUG [Thread-2555 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:03:20,831 DEBUG [Thread-2555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:20,831 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:03:20,832 INFO [Thread-2561 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2561. 2024-12-06T16:03:20,832 DEBUG [Thread-2561 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:03:20,832 DEBUG [Thread-2561 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:20,832 DEBUG [Thread-2561 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:20,832 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:03:20,833 INFO [Thread-2559 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2559. 2024-12-06T16:03:20,833 DEBUG [Thread-2559 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:03:20,833 DEBUG [Thread-2559 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:20,833 DEBUG [Thread-2555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:20,833 DEBUG [Thread-2559 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:20,833 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:03:20,835 INFO [Thread-2557 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2557. 2024-12-06T16:03:20,836 DEBUG [Thread-2557 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:03:20,836 DEBUG [Thread-2557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:20,836 DEBUG [Thread-2557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:20,836 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:03:20,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-06T16:03:20,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:20,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:20,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:20,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:20,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:20,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:20,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:20,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:20,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:20,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46850 deadline: 1733501060852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:20,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46808 deadline: 1733501060852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:20,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46864 deadline: 1733501060852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:20,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,853 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:20,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:20,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:20,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:20,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46800 deadline: 1733501060862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:20,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-06T16:03:20,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46784 deadline: 1733501060872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 2024-12-06T16:03:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 , the old value is region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-06T16:03:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., hostname=85bef17d9292,46105,1733500827732, seqNum=5 because the exception is null or not the one we care about 2024-12-06T16:03:20,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-06T16:03:20,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:20,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:20,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:20,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:20,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:20,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:21,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-06T16:03:21,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/eb9965b9306e4efe802651895037f4ea 2024-12-06T16:03:21,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/4d830adc21674e06a48c973b274650b2 is 50, key is test_row_0/C:col10/1733501000190/Put/seqid=0 2024-12-06T16:03:21,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742506_1682 (size=12301) 2024-12-06T16:03:21,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/4d830adc21674e06a48c973b274650b2 2024-12-06T16:03:21,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-06T16:03:21,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:21,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:21,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:21,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:21,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:21,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T16:03:21,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/3cfe12b939cb4100a0fff7b397935612 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/3cfe12b939cb4100a0fff7b397935612 2024-12-06T16:03:21,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/3cfe12b939cb4100a0fff7b397935612, entries=200, sequenceid=375, filesize=39.0 K 2024-12-06T16:03:21,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/eb9965b9306e4efe802651895037f4ea as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/eb9965b9306e4efe802651895037f4ea 2024-12-06T16:03:21,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/eb9965b9306e4efe802651895037f4ea, entries=150, sequenceid=375, filesize=12.0 K 2024-12-06T16:03:21,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/4d830adc21674e06a48c973b274650b2 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/4d830adc21674e06a48c973b274650b2 2024-12-06T16:03:21,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/4d830adc21674e06a48c973b274650b2, entries=150, sequenceid=375, filesize=12.0 K 2024-12-06T16:03:21,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for d143cebc3e2771774f1f66ab4723edf2 in 970ms, sequenceid=375, compaction requested=true 2024-12-06T16:03:21,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:21,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d143cebc3e2771774f1f66ab4723edf2:A, priority=-2147483648, current under compaction store size is 1 2024-12-06T16:03:21,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:21,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d143cebc3e2771774f1f66ab4723edf2:B, priority=-2147483648, current under compaction store size is 2 2024-12-06T16:03:21,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:21,166 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:03:21,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d143cebc3e2771774f1f66ab4723edf2:C, priority=-2147483648, current under compaction store size is 3 2024-12-06T16:03:21,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:03:21,166 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:03:21,167 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:03:21,167 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:03:21,167 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): d143cebc3e2771774f1f66ab4723edf2/B is initiating minor compaction (all files) 2024-12-06T16:03:21,167 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1541): d143cebc3e2771774f1f66ab4723edf2/A is initiating minor compaction (all files) 2024-12-06T16:03:21,167 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d143cebc3e2771774f1f66ab4723edf2/B in TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:21,167 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d143cebc3e2771774f1f66ab4723edf2/A in TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:21,167 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/60b619c45d5a4336878ed0a0dc96e5e4, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/815f1befa7874017b38d70238c6bb7d6, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/eb9965b9306e4efe802651895037f4ea] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp, totalSize=36.8 K 2024-12-06T16:03:21,167 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/28b65d67c21f496c9eff4eccc1bd2154, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/4b35fe7b83654882b9627b6c49d0535c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/3cfe12b939cb4100a0fff7b397935612] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp, totalSize=100.9 K 2024-12-06T16:03:21,167 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:21,167 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. files: [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/28b65d67c21f496c9eff4eccc1bd2154, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/4b35fe7b83654882b9627b6c49d0535c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/3cfe12b939cb4100a0fff7b397935612] 2024-12-06T16:03:21,167 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 60b619c45d5a4336878ed0a0dc96e5e4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733500997717 2024-12-06T16:03:21,168 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28b65d67c21f496c9eff4eccc1bd2154, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733500997717 2024-12-06T16:03:21,168 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 815f1befa7874017b38d70238c6bb7d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733500998378 2024-12-06T16:03:21,168 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b35fe7b83654882b9627b6c49d0535c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733500998378 2024-12-06T16:03:21,168 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting eb9965b9306e4efe802651895037f4ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1733501000187 2024-12-06T16:03:21,168 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cfe12b939cb4100a0fff7b397935612, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1733501000187 2024-12-06T16:03:21,174 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:21,176 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d143cebc3e2771774f1f66ab4723edf2#B#compaction#576 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:03:21,177 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/2629fa7c664c4a10947e7f855194c6b0 is 50, key is test_row_0/B:col10/1733501000190/Put/seqid=0 2024-12-06T16:03:21,178 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241206b24a5214cf5f4b4e913a54f6d1d0a81e_d143cebc3e2771774f1f66ab4723edf2 store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:21,181 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241206b24a5214cf5f4b4e913a54f6d1d0a81e_d143cebc3e2771774f1f66ab4723edf2, store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:21,181 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241206b24a5214cf5f4b4e913a54f6d1d0a81e_d143cebc3e2771774f1f66ab4723edf2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:21,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742507_1683 (size=13221) 2024-12-06T16:03:21,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742508_1684 (size=4469) 2024-12-06T16:03:21,190 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/2629fa7c664c4a10947e7f855194c6b0 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/2629fa7c664c4a10947e7f855194c6b0 2024-12-06T16:03:21,194 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d143cebc3e2771774f1f66ab4723edf2/B of d143cebc3e2771774f1f66ab4723edf2 into 2629fa7c664c4a10947e7f855194c6b0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:03:21,194 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:21,194 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., storeName=d143cebc3e2771774f1f66ab4723edf2/B, priority=13, startTime=1733501001166; duration=0sec 2024-12-06T16:03:21,194 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-06T16:03:21,194 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d143cebc3e2771774f1f66ab4723edf2:B 2024-12-06T16:03:21,194 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-06T16:03:21,195 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-06T16:03:21,195 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1541): d143cebc3e2771774f1f66ab4723edf2/C is initiating minor compaction (all files) 2024-12-06T16:03:21,195 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d143cebc3e2771774f1f66ab4723edf2/C in TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:21,195 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/811fad4776394d999817203763360d8f, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/a4a7fe1386024ac6b3984ca83d53e8e7, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/4d830adc21674e06a48c973b274650b2] into tmpdir=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp, totalSize=36.8 K 2024-12-06T16:03:21,195 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 811fad4776394d999817203763360d8f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733500997717 2024-12-06T16:03:21,195 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting a4a7fe1386024ac6b3984ca83d53e8e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733500998378 2024-12-06T16:03:21,196 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d830adc21674e06a48c973b274650b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1733501000187 2024-12-06T16:03:21,201 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d143cebc3e2771774f1f66ab4723edf2#C#compaction#577 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:03:21,201 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/fa5920716eb14a1482ebdd9cdab4899f is 50, key is test_row_0/C:col10/1733501000190/Put/seqid=0 2024-12-06T16:03:21,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742509_1685 (size=13221) 2024-12-06T16:03:21,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-06T16:03:21,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:21,303 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing d143cebc3e2771774f1f66ab4723edf2 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-12-06T16:03:21,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120606676dfbcce94f7aa006cb306c38da21_d143cebc3e2771774f1f66ab4723edf2 is 50, key is test_row_0/A:col10/1733501000214/Put/seqid=0 2024-12-06T16:03:21,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-06T16:03:21,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742510_1686 (size=12454) 2024-12-06T16:03:21,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(8855): Flush requested on d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:21,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. as already flushing 2024-12-06T16:03:21,364 INFO [Thread-2544 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2544. 2024-12-06T16:03:21,364 DEBUG [Thread-2544 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:03:21,364 DEBUG [Thread-2544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:21,364 DEBUG [Thread-2544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:21,364 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:03:21,365 INFO [Thread-2548 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2548. 2024-12-06T16:03:21,365 DEBUG [Thread-2548 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:03:21,365 DEBUG [Thread-2548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:21,365 DEBUG [Thread-2548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:21,365 INFO [Thread-2542 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2542. 2024-12-06T16:03:21,365 DEBUG [Thread-2542 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:03:21,365 DEBUG [Thread-2542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:21,365 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:03:21,365 DEBUG [Thread-2542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:21,365 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:03:21,373 INFO [Thread-2546 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2546. 2024-12-06T16:03:21,373 DEBUG [Thread-2546 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:03:21,373 DEBUG [Thread-2546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:21,373 DEBUG [Thread-2546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:21,373 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:03:21,383 INFO [Thread-2550 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2550. 2024-12-06T16:03:21,383 DEBUG [Thread-2550 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-06T16:03:21,383 DEBUG [Thread-2550 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:21,383 DEBUG [Thread-2550 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:21,383 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:03:21,587 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d143cebc3e2771774f1f66ab4723edf2#A#compaction#575 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T16:03:21,588 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/8d497517e4a64fc99904d321582bb65f is 175, key is test_row_0/A:col10/1733501000190/Put/seqid=0 2024-12-06T16:03:21,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742511_1687 (size=32175) 2024-12-06T16:03:21,608 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/fa5920716eb14a1482ebdd9cdab4899f as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/fa5920716eb14a1482ebdd9cdab4899f 2024-12-06T16:03:21,612 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d143cebc3e2771774f1f66ab4723edf2/C of d143cebc3e2771774f1f66ab4723edf2 into fa5920716eb14a1482ebdd9cdab4899f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:03:21,613 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:21,613 INFO [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., storeName=d143cebc3e2771774f1f66ab4723edf2/C, priority=13, startTime=1733501001166; duration=0sec 2024-12-06T16:03:21,613 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:21,613 DEBUG [RS:0;85bef17d9292:46105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d143cebc3e2771774f1f66ab4723edf2:C 2024-12-06T16:03:21,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:21,715 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120606676dfbcce94f7aa006cb306c38da21_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120606676dfbcce94f7aa006cb306c38da21_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:21,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/b04c578b873548b8b3276d3a4ba5c70e, store: [table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:21,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/b04c578b873548b8b3276d3a4ba5c70e is 175, key is test_row_0/A:col10/1733501000214/Put/seqid=0 2024-12-06T16:03:21,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742512_1688 (size=31255) 2024-12-06T16:03:21,720 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=393, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/b04c578b873548b8b3276d3a4ba5c70e 2024-12-06T16:03:21,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/aa1c35bb00a04f899c11eedd843985e5 is 50, key is test_row_0/B:col10/1733501000214/Put/seqid=0 2024-12-06T16:03:21,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742513_1689 (size=12301) 2024-12-06T16:03:21,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-06T16:03:21,995 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/8d497517e4a64fc99904d321582bb65f as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/8d497517e4a64fc99904d321582bb65f 2024-12-06T16:03:21,999 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d143cebc3e2771774f1f66ab4723edf2/A of d143cebc3e2771774f1f66ab4723edf2 into 8d497517e4a64fc99904d321582bb65f(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T16:03:21,999 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:21,999 INFO [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., storeName=d143cebc3e2771774f1f66ab4723edf2/A, priority=13, startTime=1733501001166; duration=0sec 2024-12-06T16:03:21,999 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-06T16:03:21,999 DEBUG [RS:0;85bef17d9292:46105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d143cebc3e2771774f1f66ab4723edf2:A 2024-12-06T16:03:22,128 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/aa1c35bb00a04f899c11eedd843985e5 2024-12-06T16:03:22,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/97e91c14addc47eb9127b43e41035240 is 50, key is test_row_0/C:col10/1733501000214/Put/seqid=0 2024-12-06T16:03:22,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742514_1690 (size=12301) 2024-12-06T16:03:22,537 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/97e91c14addc47eb9127b43e41035240 2024-12-06T16:03:22,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/b04c578b873548b8b3276d3a4ba5c70e as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/b04c578b873548b8b3276d3a4ba5c70e 2024-12-06T16:03:22,543 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/b04c578b873548b8b3276d3a4ba5c70e, entries=150, sequenceid=393, filesize=30.5 K 2024-12-06T16:03:22,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/aa1c35bb00a04f899c11eedd843985e5 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/aa1c35bb00a04f899c11eedd843985e5 2024-12-06T16:03:22,546 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/aa1c35bb00a04f899c11eedd843985e5, entries=150, sequenceid=393, filesize=12.0 K 2024-12-06T16:03:22,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/97e91c14addc47eb9127b43e41035240 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/97e91c14addc47eb9127b43e41035240 2024-12-06T16:03:22,550 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/97e91c14addc47eb9127b43e41035240, entries=150, sequenceid=393, filesize=12.0 K 2024-12-06T16:03:22,550 INFO [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=33.54 KB/34350 for d143cebc3e2771774f1f66ab4723edf2 in 1247ms, sequenceid=393, compaction requested=false 2024-12-06T16:03:22,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for d143cebc3e2771774f1f66ab4723edf2: 2024-12-06T16:03:22,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:22,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/85bef17d9292:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-06T16:03:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-06T16:03:22,553 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-06T16:03:22,553 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8640 sec 2024-12-06T16:03:22,554 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.8690 sec 2024-12-06T16:03:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-06T16:03:22,832 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:03:22,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-06T16:03:22,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-12-06T16:03:22,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-12-06T16:03:22,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-12-06T16:03:22,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-12-06T16:03:22,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-12-06T16:03:22,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-06T16:03:22,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2867 2024-12-06T16:03:22,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2829 2024-12-06T16:03:22,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2760 2024-12-06T16:03:22,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2873 2024-12-06T16:03:22,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2834 2024-12-06T16:03:22,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-06T16:03:22,832 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T16:03:22,833 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T16:03:22,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:22,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:22,833 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:03:22,833 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-06T16:03:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-06T16:03:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-06T16:03:22,836 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733501002836"}]},"ts":"1733501002836"} 2024-12-06T16:03:22,837 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-06T16:03:22,838 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-06T16:03:22,838 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-06T16:03:22,839 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d143cebc3e2771774f1f66ab4723edf2, UNASSIGN}] 2024-12-06T16:03:22,840 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d143cebc3e2771774f1f66ab4723edf2, UNASSIGN 2024-12-06T16:03:22,840 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=d143cebc3e2771774f1f66ab4723edf2, regionState=CLOSING, regionLocation=85bef17d9292,46105,1733500827732 2024-12-06T16:03:22,841 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d143cebc3e2771774f1f66ab4723edf2, UNASSIGN because future has completed 2024-12-06T16:03:22,842 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-06T16:03:22,842 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732}] 2024-12-06T16:03:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-06T16:03:22,994 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(122): Close d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:22,994 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-06T16:03:22,994 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1722): Closing d143cebc3e2771774f1f66ab4723edf2, disabling compactions & flushes 2024-12-06T16:03:22,994 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:22,994 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:22,994 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. after waiting 0 ms 2024-12-06T16:03:22,994 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:22,994 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(2902): Flushing d143cebc3e2771774f1f66ab4723edf2 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-12-06T16:03:22,999 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412062020fdc125084fc7b0eb7f433a2d3063_d143cebc3e2771774f1f66ab4723edf2 is 50, key is test_row_0/A:col10/1733501001382/Put/seqid=0 2024-12-06T16:03:23,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742515_1691 (size=9914) 2024-12-06T16:03:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-06T16:03:23,403 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T16:03:23,406 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412062020fdc125084fc7b0eb7f433a2d3063_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412062020fdc125084fc7b0eb7f433a2d3063_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:23,407 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/1d28bebcf4224a3d93bc9bb664f46961, store: [table=TestAcidGuarantees family=A region=d143cebc3e2771774f1f66ab4723edf2] 2024-12-06T16:03:23,407 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/1d28bebcf4224a3d93bc9bb664f46961 is 175, key is test_row_0/A:col10/1733501001382/Put/seqid=0 2024-12-06T16:03:23,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742516_1692 (size=22561) 2024-12-06T16:03:23,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-06T16:03:23,811 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=403, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/1d28bebcf4224a3d93bc9bb664f46961 2024-12-06T16:03:23,817 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/07635943694c4adda71f8eef08a46ddb is 50, key is test_row_0/B:col10/1733501001382/Put/seqid=0 2024-12-06T16:03:23,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742517_1693 (size=9857) 2024-12-06T16:03:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-06T16:03:24,220 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/07635943694c4adda71f8eef08a46ddb 2024-12-06T16:03:24,226 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/0dfa57ebe2e34b4fa891d288b3010be0 is 50, key is test_row_0/C:col10/1733501001382/Put/seqid=0 2024-12-06T16:03:24,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742518_1694 (size=9857) 2024-12-06T16:03:24,629 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/0dfa57ebe2e34b4fa891d288b3010be0 2024-12-06T16:03:24,632 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/A/1d28bebcf4224a3d93bc9bb664f46961 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/1d28bebcf4224a3d93bc9bb664f46961 2024-12-06T16:03:24,635 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/1d28bebcf4224a3d93bc9bb664f46961, entries=100, sequenceid=403, filesize=22.0 K 2024-12-06T16:03:24,636 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/B/07635943694c4adda71f8eef08a46ddb as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/07635943694c4adda71f8eef08a46ddb 2024-12-06T16:03:24,638 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/07635943694c4adda71f8eef08a46ddb, entries=100, sequenceid=403, filesize=9.6 K 2024-12-06T16:03:24,639 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/.tmp/C/0dfa57ebe2e34b4fa891d288b3010be0 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/0dfa57ebe2e34b4fa891d288b3010be0 2024-12-06T16:03:24,641 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/0dfa57ebe2e34b4fa891d288b3010be0, entries=100, sequenceid=403, filesize=9.6 K 2024-12-06T16:03:24,642 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for d143cebc3e2771774f1f66ab4723edf2 in 1648ms, sequenceid=403, compaction requested=true 2024-12-06T16:03:24,643 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/be4ecd92684c4969bef5a96f6da86995, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/c864a0a43e884f05a1bcf6e434e62249, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/e7ec001b72e94b5cbe142d56acb7a3fb, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/8e095af1dbfe4759917be065681b9c55, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/aaead29bdced4206b1ac7fc9e775833c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/111bfa18bf7148d8bf08efbabc5e4fa3, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/692ec80c1d0542159b97a9c29eb85720, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/a2e7344100864d2189827808653ba5f8, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/e5cd51d8e5f2487cae978b7121f0c86b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/ea053a8a453d4ae194ef4970860b3aa6, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/d420dae89c504f209c3cc38949e29859, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/f41b6b74269b47149044ac99be1b8ab0, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/203e8a27442b48ed9735c54107598d42, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/360f4e9d90094ad99fc27bdd3aca1feb, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/fb4a54d3b3034049b68da506a45cd530, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/1cfa9013643a45e599ee57541ce0e3b2, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/279fe5be81d84caeabd2d2ffaecfd157, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/a24f5292c83c4f22a7b875eeadb0cb80, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/6fbca934cdb94eaa86e4f737e1e4403d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/e9344b29af88435597450c097523b075, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/7b3fa049fe5842569ea766049cf60c4e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/e64b41924b7b44a2b3a4606c4cdf2ef2, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/84deb76d45a1440b8213af9c2a9b3239, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/28b65d67c21f496c9eff4eccc1bd2154, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/8f5f3b340f8a4ef69cb74085ed83928b, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/4b35fe7b83654882b9627b6c49d0535c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/3cfe12b939cb4100a0fff7b397935612] to archive 2024-12-06T16:03:24,643 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T16:03:24,645 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/be4ecd92684c4969bef5a96f6da86995 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/be4ecd92684c4969bef5a96f6da86995 2024-12-06T16:03:24,645 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/c864a0a43e884f05a1bcf6e434e62249 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/c864a0a43e884f05a1bcf6e434e62249 2024-12-06T16:03:24,646 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/e7ec001b72e94b5cbe142d56acb7a3fb to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/e7ec001b72e94b5cbe142d56acb7a3fb 2024-12-06T16:03:24,647 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/8e095af1dbfe4759917be065681b9c55 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/8e095af1dbfe4759917be065681b9c55 2024-12-06T16:03:24,648 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/aaead29bdced4206b1ac7fc9e775833c to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/aaead29bdced4206b1ac7fc9e775833c 2024-12-06T16:03:24,648 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/111bfa18bf7148d8bf08efbabc5e4fa3 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/111bfa18bf7148d8bf08efbabc5e4fa3 2024-12-06T16:03:24,649 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/692ec80c1d0542159b97a9c29eb85720 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/692ec80c1d0542159b97a9c29eb85720 2024-12-06T16:03:24,650 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/a2e7344100864d2189827808653ba5f8 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/a2e7344100864d2189827808653ba5f8 2024-12-06T16:03:24,651 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/e5cd51d8e5f2487cae978b7121f0c86b to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/e5cd51d8e5f2487cae978b7121f0c86b 2024-12-06T16:03:24,652 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/ea053a8a453d4ae194ef4970860b3aa6 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/ea053a8a453d4ae194ef4970860b3aa6 2024-12-06T16:03:24,652 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/d420dae89c504f209c3cc38949e29859 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/d420dae89c504f209c3cc38949e29859 2024-12-06T16:03:24,653 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/f41b6b74269b47149044ac99be1b8ab0 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/f41b6b74269b47149044ac99be1b8ab0 2024-12-06T16:03:24,654 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/203e8a27442b48ed9735c54107598d42 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/203e8a27442b48ed9735c54107598d42 2024-12-06T16:03:24,655 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/360f4e9d90094ad99fc27bdd3aca1feb to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/360f4e9d90094ad99fc27bdd3aca1feb 2024-12-06T16:03:24,655 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/fb4a54d3b3034049b68da506a45cd530 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/fb4a54d3b3034049b68da506a45cd530 2024-12-06T16:03:24,656 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/1cfa9013643a45e599ee57541ce0e3b2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/1cfa9013643a45e599ee57541ce0e3b2 2024-12-06T16:03:24,657 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/279fe5be81d84caeabd2d2ffaecfd157 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/279fe5be81d84caeabd2d2ffaecfd157 2024-12-06T16:03:24,657 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/a24f5292c83c4f22a7b875eeadb0cb80 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/a24f5292c83c4f22a7b875eeadb0cb80 2024-12-06T16:03:24,658 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/6fbca934cdb94eaa86e4f737e1e4403d to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/6fbca934cdb94eaa86e4f737e1e4403d 2024-12-06T16:03:24,659 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/e9344b29af88435597450c097523b075 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/e9344b29af88435597450c097523b075 2024-12-06T16:03:24,660 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/7b3fa049fe5842569ea766049cf60c4e to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/7b3fa049fe5842569ea766049cf60c4e 2024-12-06T16:03:24,661 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/e64b41924b7b44a2b3a4606c4cdf2ef2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/e64b41924b7b44a2b3a4606c4cdf2ef2 2024-12-06T16:03:24,661 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/84deb76d45a1440b8213af9c2a9b3239 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/84deb76d45a1440b8213af9c2a9b3239 2024-12-06T16:03:24,662 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/28b65d67c21f496c9eff4eccc1bd2154 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/28b65d67c21f496c9eff4eccc1bd2154 2024-12-06T16:03:24,663 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/8f5f3b340f8a4ef69cb74085ed83928b to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/8f5f3b340f8a4ef69cb74085ed83928b 2024-12-06T16:03:24,664 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/4b35fe7b83654882b9627b6c49d0535c to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/4b35fe7b83654882b9627b6c49d0535c 2024-12-06T16:03:24,664 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/3cfe12b939cb4100a0fff7b397935612 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/3cfe12b939cb4100a0fff7b397935612 2024-12-06T16:03:24,666 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/2e99081ebf59449d974a64223f3838bd, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/d1a727d8f9e040ada48a6b07e19a83f8, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/925005f3e4cb48e58b6c9a1e02a2b779, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/c53a97f238e547f687bb8e278f51e9cf, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/7b6edebe472b408dacfed7db7b633ea7, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/fd737c3639e74ff7b0821b83a8662d1d, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/3f87e4357ca54f9683339da93e460820, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/2a1db293552e4b3e9f74c5232377cdd1, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/eb1171617e084072b84c7260daaeb58e, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/b03a88f93a9846d4a40301788b0cc79c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/14442bf624344c27bfd2255ae5fe7cc8, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/964582024b7c47b593ce7e1a6d1652db, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/45aa5cc7785e47a6a77cabc0f885d359, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/e8616b8ef2094309a431a0b4392766eb, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/56de9e083df940cdb487c9d45b94673a, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/66837551ad964e4991df1d05075b2fd1, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/ffd61fda1c4d4efa8875ea6ae26c6e78, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/8487523788f44db0bdcdd125985efa6c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/5863d13f1ca64619aeacb9005e5834cd, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/67b912624ff84ea2b1e72476056b9628, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/66af06dd8b1641b6a30ccbf83b5525fa, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/a942aebedecc43a599c226a14f480add, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/b0e658bd3f6c4749abe25937190f6021, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/60b619c45d5a4336878ed0a0dc96e5e4, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/1a2e038c60d6481aa6054c2d741d6e78, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/815f1befa7874017b38d70238c6bb7d6, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/eb9965b9306e4efe802651895037f4ea] to archive 2024-12-06T16:03:24,666 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T16:03:24,667 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/2e99081ebf59449d974a64223f3838bd to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/2e99081ebf59449d974a64223f3838bd 2024-12-06T16:03:24,668 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/d1a727d8f9e040ada48a6b07e19a83f8 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/d1a727d8f9e040ada48a6b07e19a83f8 2024-12-06T16:03:24,669 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/925005f3e4cb48e58b6c9a1e02a2b779 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/925005f3e4cb48e58b6c9a1e02a2b779 2024-12-06T16:03:24,670 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/c53a97f238e547f687bb8e278f51e9cf to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/c53a97f238e547f687bb8e278f51e9cf 2024-12-06T16:03:24,670 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/7b6edebe472b408dacfed7db7b633ea7 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/7b6edebe472b408dacfed7db7b633ea7 2024-12-06T16:03:24,671 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/fd737c3639e74ff7b0821b83a8662d1d to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/fd737c3639e74ff7b0821b83a8662d1d 2024-12-06T16:03:24,672 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/3f87e4357ca54f9683339da93e460820 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/3f87e4357ca54f9683339da93e460820 2024-12-06T16:03:24,673 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/2a1db293552e4b3e9f74c5232377cdd1 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/2a1db293552e4b3e9f74c5232377cdd1 2024-12-06T16:03:24,673 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/eb1171617e084072b84c7260daaeb58e to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/eb1171617e084072b84c7260daaeb58e 2024-12-06T16:03:24,674 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/b03a88f93a9846d4a40301788b0cc79c to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/b03a88f93a9846d4a40301788b0cc79c 2024-12-06T16:03:24,675 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/14442bf624344c27bfd2255ae5fe7cc8 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/14442bf624344c27bfd2255ae5fe7cc8 2024-12-06T16:03:24,676 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/964582024b7c47b593ce7e1a6d1652db to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/964582024b7c47b593ce7e1a6d1652db 2024-12-06T16:03:24,677 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/45aa5cc7785e47a6a77cabc0f885d359 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/45aa5cc7785e47a6a77cabc0f885d359 2024-12-06T16:03:24,678 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/e8616b8ef2094309a431a0b4392766eb to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/e8616b8ef2094309a431a0b4392766eb 2024-12-06T16:03:24,678 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/56de9e083df940cdb487c9d45b94673a to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/56de9e083df940cdb487c9d45b94673a 2024-12-06T16:03:24,679 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/66837551ad964e4991df1d05075b2fd1 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/66837551ad964e4991df1d05075b2fd1 2024-12-06T16:03:24,680 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/ffd61fda1c4d4efa8875ea6ae26c6e78 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/ffd61fda1c4d4efa8875ea6ae26c6e78 2024-12-06T16:03:24,681 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/8487523788f44db0bdcdd125985efa6c to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/8487523788f44db0bdcdd125985efa6c 2024-12-06T16:03:24,682 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/5863d13f1ca64619aeacb9005e5834cd to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/5863d13f1ca64619aeacb9005e5834cd 2024-12-06T16:03:24,683 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/67b912624ff84ea2b1e72476056b9628 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/67b912624ff84ea2b1e72476056b9628 2024-12-06T16:03:24,684 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/66af06dd8b1641b6a30ccbf83b5525fa to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/66af06dd8b1641b6a30ccbf83b5525fa 2024-12-06T16:03:24,688 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/a942aebedecc43a599c226a14f480add to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/a942aebedecc43a599c226a14f480add 2024-12-06T16:03:24,692 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/b0e658bd3f6c4749abe25937190f6021 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/b0e658bd3f6c4749abe25937190f6021 2024-12-06T16:03:24,693 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/60b619c45d5a4336878ed0a0dc96e5e4 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/60b619c45d5a4336878ed0a0dc96e5e4 2024-12-06T16:03:24,696 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/1a2e038c60d6481aa6054c2d741d6e78 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/1a2e038c60d6481aa6054c2d741d6e78 2024-12-06T16:03:24,697 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/815f1befa7874017b38d70238c6bb7d6 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/815f1befa7874017b38d70238c6bb7d6 2024-12-06T16:03:24,699 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/eb9965b9306e4efe802651895037f4ea to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/eb9965b9306e4efe802651895037f4ea 2024-12-06T16:03:24,710 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/59890321bc504a39b6dd32210ad7a559, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/445143ec4970402b8d17435cf1ff3afc, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/8e29eb5b071e4896b13afe1274343fe9, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/b6254c037d0d43a2957aa64b2e7bebd3, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/3aab86a02fc24b92acb08b8f921c3efb, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/8725968be9274f3f878127aed2212a5c, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/cf40a81d832b493193f66bf0fd54a2d4, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/89ea38abeb1f4f349422c256284058c5, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/a5d3927b7ae84dac9ca3f4c8df4f2129, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/744e1ed3c33f4193a75bfe7f2c402132, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/820b4be98bf74f039b00f78e56e9f1b2, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/8474e40e62b843d8a883532db748eb74, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/0cca58e36ce540278790e649f6c03805, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/c2497e3b1aa8461980b5ac26c28e3fd1, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/3fe797a0ee6a4cd19e2433ad7c303339, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/81ceed5a49cf489387367f6b3c76fb57, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/0bd70f69e94042b285cedb9a0694b2b4, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/db8b5e15b1534591b2af62eafda6d3f2, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/c617331ac0a042c6bbc641fb20fd1ebe, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/2672d0e222c94944b7484b88768a7e85, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/9b7f54c3d6f943f5b2b48ef78aa13d58, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/23bf097b587a4723b05d5b0ce1b72204, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/8b415e7a6778462288cd27773fcf0b35, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/811fad4776394d999817203763360d8f, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/d898dc57982347e1a7b6610081312c62, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/a4a7fe1386024ac6b3984ca83d53e8e7, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/4d830adc21674e06a48c973b274650b2] to archive 2024-12-06T16:03:24,711 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T16:03:24,713 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/59890321bc504a39b6dd32210ad7a559 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/59890321bc504a39b6dd32210ad7a559 2024-12-06T16:03:24,714 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/445143ec4970402b8d17435cf1ff3afc to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/445143ec4970402b8d17435cf1ff3afc 2024-12-06T16:03:24,715 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/8e29eb5b071e4896b13afe1274343fe9 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/8e29eb5b071e4896b13afe1274343fe9 2024-12-06T16:03:24,716 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/b6254c037d0d43a2957aa64b2e7bebd3 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/b6254c037d0d43a2957aa64b2e7bebd3 2024-12-06T16:03:24,717 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/3aab86a02fc24b92acb08b8f921c3efb to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/3aab86a02fc24b92acb08b8f921c3efb 2024-12-06T16:03:24,719 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/8725968be9274f3f878127aed2212a5c to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/8725968be9274f3f878127aed2212a5c 2024-12-06T16:03:24,720 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/cf40a81d832b493193f66bf0fd54a2d4 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/cf40a81d832b493193f66bf0fd54a2d4 2024-12-06T16:03:24,720 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/89ea38abeb1f4f349422c256284058c5 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/89ea38abeb1f4f349422c256284058c5 2024-12-06T16:03:24,721 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/a5d3927b7ae84dac9ca3f4c8df4f2129 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/a5d3927b7ae84dac9ca3f4c8df4f2129 2024-12-06T16:03:24,722 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/744e1ed3c33f4193a75bfe7f2c402132 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/744e1ed3c33f4193a75bfe7f2c402132 2024-12-06T16:03:24,723 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/820b4be98bf74f039b00f78e56e9f1b2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/820b4be98bf74f039b00f78e56e9f1b2 2024-12-06T16:03:24,724 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/8474e40e62b843d8a883532db748eb74 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/8474e40e62b843d8a883532db748eb74 2024-12-06T16:03:24,725 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/0cca58e36ce540278790e649f6c03805 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/0cca58e36ce540278790e649f6c03805 2024-12-06T16:03:24,726 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/c2497e3b1aa8461980b5ac26c28e3fd1 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/c2497e3b1aa8461980b5ac26c28e3fd1 2024-12-06T16:03:24,727 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/3fe797a0ee6a4cd19e2433ad7c303339 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/3fe797a0ee6a4cd19e2433ad7c303339 2024-12-06T16:03:24,727 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/81ceed5a49cf489387367f6b3c76fb57 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/81ceed5a49cf489387367f6b3c76fb57 2024-12-06T16:03:24,728 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/0bd70f69e94042b285cedb9a0694b2b4 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/0bd70f69e94042b285cedb9a0694b2b4 2024-12-06T16:03:24,729 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/db8b5e15b1534591b2af62eafda6d3f2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/db8b5e15b1534591b2af62eafda6d3f2 2024-12-06T16:03:24,730 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/c617331ac0a042c6bbc641fb20fd1ebe to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/c617331ac0a042c6bbc641fb20fd1ebe 2024-12-06T16:03:24,731 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/2672d0e222c94944b7484b88768a7e85 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/2672d0e222c94944b7484b88768a7e85 2024-12-06T16:03:24,732 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/9b7f54c3d6f943f5b2b48ef78aa13d58 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/9b7f54c3d6f943f5b2b48ef78aa13d58 2024-12-06T16:03:24,733 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/23bf097b587a4723b05d5b0ce1b72204 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/23bf097b587a4723b05d5b0ce1b72204 2024-12-06T16:03:24,734 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/8b415e7a6778462288cd27773fcf0b35 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/8b415e7a6778462288cd27773fcf0b35 2024-12-06T16:03:24,735 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/811fad4776394d999817203763360d8f to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/811fad4776394d999817203763360d8f 2024-12-06T16:03:24,735 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/d898dc57982347e1a7b6610081312c62 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/d898dc57982347e1a7b6610081312c62 2024-12-06T16:03:24,736 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/a4a7fe1386024ac6b3984ca83d53e8e7 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/a4a7fe1386024ac6b3984ca83d53e8e7 2024-12-06T16:03:24,737 DEBUG [StoreCloser-TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/4d830adc21674e06a48c973b274650b2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/4d830adc21674e06a48c973b274650b2 2024-12-06T16:03:24,741 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/recovered.edits/406.seqid, newMaxSeqId=406, maxSeqId=4 2024-12-06T16:03:24,741 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. 2024-12-06T16:03:24,741 DEBUG [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1676): Region close journal for d143cebc3e2771774f1f66ab4723edf2: Waiting for close lock at 1733501002994Running coprocessor pre-close hooks at 1733501002994Disabling compacts and flushes for region at 1733501002994Disabling writes for close at 1733501002994Obtaining lock to block concurrent updates at 1733501002994Preparing flush snapshotting stores in d143cebc3e2771774f1f66ab4723edf2 at 1733501002994Finished memstore snapshotting TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1733501002994Flushing stores of TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2. at 1733501002995 (+1 ms)Flushing d143cebc3e2771774f1f66ab4723edf2/A: creating writer at 1733501002995Flushing mob file d143cebc3e2771774f1f66ab4723edf2/A: appending metadata at 1733501002999 (+4 ms)Flushing mob file d143cebc3e2771774f1f66ab4723edf2/A: closing flushed file at 1733501002999Flushing d143cebc3e2771774f1f66ab4723edf2/A: appending metadata at 1733501003407 (+408 ms)Flushing d143cebc3e2771774f1f66ab4723edf2/A: closing flushed file at 1733501003407Flushing d143cebc3e2771774f1f66ab4723edf2/B: creating writer at 1733501003814 (+407 ms)Flushing d143cebc3e2771774f1f66ab4723edf2/B: appending metadata at 1733501003816 (+2 ms)Flushing d143cebc3e2771774f1f66ab4723edf2/B: closing flushed file at 1733501003816Flushing d143cebc3e2771774f1f66ab4723edf2/C: creating writer at 1733501004223 (+407 ms)Flushing d143cebc3e2771774f1f66ab4723edf2/C: appending metadata at 1733501004225 (+2 ms)Flushing d143cebc3e2771774f1f66ab4723edf2/C: closing flushed file at 1733501004225Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4bbf2d22: reopening flushed file at 1733501004632 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c21d7c1: reopening flushed file at 1733501004635 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39f7c77a: reopening flushed file at 1733501004638 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for d143cebc3e2771774f1f66ab4723edf2 in 1648ms, sequenceid=403, compaction requested=true at 1733501004642 (+4 ms)Writing region close event to WAL at 1733501004738 (+96 ms)Running coprocessor post-close hooks at 1733501004741 (+3 ms)Closed at 1733501004741 2024-12-06T16:03:24,743 INFO [RS_CLOSE_REGION-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(157): Closed d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:24,744 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=d143cebc3e2771774f1f66ab4723edf2, regionState=CLOSED 2024-12-06T16:03:24,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 because future has completed 2024-12-06T16:03:24,747 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-06T16:03:24,747 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseRegionProcedure d143cebc3e2771774f1f66ab4723edf2, server=85bef17d9292,46105,1733500827732 in 1.9040 sec 2024-12-06T16:03:24,748 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-12-06T16:03:24,748 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d143cebc3e2771774f1f66ab4723edf2, UNASSIGN in 1.9080 sec 2024-12-06T16:03:24,750 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-06T16:03:24,750 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9110 sec 2024-12-06T16:03:24,751 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733501004751"}]},"ts":"1733501004751"} 2024-12-06T16:03:24,752 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-06T16:03:24,752 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-06T16:03:24,754 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9200 sec 2024-12-06T16:03:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-06T16:03:24,982 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:03:24,983 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-06T16:03:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:03:24,984 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:03:24,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-06T16:03:24,985 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=166, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:03:24,987 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:24,989 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A, FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B, FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C, FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/recovered.edits] 2024-12-06T16:03:24,991 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/1d28bebcf4224a3d93bc9bb664f46961 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/1d28bebcf4224a3d93bc9bb664f46961 2024-12-06T16:03:24,992 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/8d497517e4a64fc99904d321582bb65f to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/8d497517e4a64fc99904d321582bb65f 2024-12-06T16:03:24,993 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/b04c578b873548b8b3276d3a4ba5c70e to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/A/b04c578b873548b8b3276d3a4ba5c70e 2024-12-06T16:03:24,994 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/07635943694c4adda71f8eef08a46ddb to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/07635943694c4adda71f8eef08a46ddb 2024-12-06T16:03:24,995 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/2629fa7c664c4a10947e7f855194c6b0 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/2629fa7c664c4a10947e7f855194c6b0 2024-12-06T16:03:24,996 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/aa1c35bb00a04f899c11eedd843985e5 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/B/aa1c35bb00a04f899c11eedd843985e5 2024-12-06T16:03:24,997 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/0dfa57ebe2e34b4fa891d288b3010be0 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/0dfa57ebe2e34b4fa891d288b3010be0 2024-12-06T16:03:24,998 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/97e91c14addc47eb9127b43e41035240 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/97e91c14addc47eb9127b43e41035240 2024-12-06T16:03:24,999 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/fa5920716eb14a1482ebdd9cdab4899f to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/C/fa5920716eb14a1482ebdd9cdab4899f 2024-12-06T16:03:25,001 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/recovered.edits/406.seqid to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2/recovered.edits/406.seqid 2024-12-06T16:03:25,001 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/default/TestAcidGuarantees/d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:25,001 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-06T16:03:25,001 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-06T16:03:25,002 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-06T16:03:25,004 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206061b740f2fd74a0db76dba68a4645daf_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206061b740f2fd74a0db76dba68a4645daf_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:25,005 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120606676dfbcce94f7aa006cb306c38da21_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120606676dfbcce94f7aa006cb306c38da21_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:25,006 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412060db15f40de014e8098267313c7328f70_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412060db15f40de014e8098267313c7328f70_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:25,007 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412061003bdf3bcbb43e4b492ce9a01b54727_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412061003bdf3bcbb43e4b492ce9a01b54727_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:25,007 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412062020fdc125084fc7b0eb7f433a2d3063_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412062020fdc125084fc7b0eb7f433a2d3063_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:25,008 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120625913b0085e8445bbb2146646dfd527f_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120625913b0085e8445bbb2146646dfd527f_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:25,009 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120632258dae56df4f1ca61b06094a3ef2ce_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120632258dae56df4f1ca61b06094a3ef2ce_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:25,010 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206326f9b597a154edf8360d401a3b27be9_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206326f9b597a154edf8360d401a3b27be9_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:25,011 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412064719fd9d5c1845d8a7ca527998902247_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412064719fd9d5c1845d8a7ca527998902247_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:25,012 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412065fe550e45a2842e0bd66362a7c505a38_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412065fe550e45a2842e0bd66362a7c505a38_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:25,013 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120661e9a7cc52164cca91b4be508f53d9cf_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120661e9a7cc52164cca91b4be508f53d9cf_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:25,014 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412067c98eae8f2fd4980bc1e8b71a35c61de_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412067c98eae8f2fd4980bc1e8b71a35c61de_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:25,015 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a234dbd68de444e5ac295595cca12061_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206a234dbd68de444e5ac295595cca12061_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:25,016 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206aa7c17f61a4941b3b7fe15deb35aa4e3_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206aa7c17f61a4941b3b7fe15deb35aa4e3_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:25,017 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206afde213d0bd54621a1682dc8b2c15606_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206afde213d0bd54621a1682dc8b2c15606_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:25,018 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206caf8516b9ec9411a9711bbdd11491267_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206caf8516b9ec9411a9711bbdd11491267_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:25,019 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206cca803cd0bf64caabaa220118512dae4_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206cca803cd0bf64caabaa220118512dae4_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:25,020 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206d6f4febb6f5444e69426776311552f55_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206d6f4febb6f5444e69426776311552f55_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:25,021 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ed090a1656b04de6b7e0ae6b980cf6c2_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206ed090a1656b04de6b7e0ae6b980cf6c2_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:25,022 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206f11c441ade244e4c9dc1fd369d9b8a87_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206f11c441ade244e4c9dc1fd369d9b8a87_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:25,023 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206fc2a2a8df63a4b568abec59fdb2d2980_d143cebc3e2771774f1f66ab4723edf2 to hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241206fc2a2a8df63a4b568abec59fdb2d2980_d143cebc3e2771774f1f66ab4723edf2 2024-12-06T16:03:25,023 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-06T16:03:25,025 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=166, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:03:25,027 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-06T16:03:25,029 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-06T16:03:25,030 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=166, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:03:25,030 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-06T16:03:25,030 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733501005030"}]},"ts":"9223372036854775807"} 2024-12-06T16:03:25,032 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-06T16:03:25,032 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d143cebc3e2771774f1f66ab4723edf2, NAME => 'TestAcidGuarantees,,1733500977699.d143cebc3e2771774f1f66ab4723edf2.', STARTKEY => '', ENDKEY => ''}] 2024-12-06T16:03:25,032 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-06T16:03:25,032 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733501005032"}]},"ts":"9223372036854775807"} 2024-12-06T16:03:25,035 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-06T16:03:25,036 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=166, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-06T16:03:25,037 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 53 msec 2024-12-06T16:03:25,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-06T16:03:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-06T16:03:25,092 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-06T16:03:25,101 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=242 (was 242), OpenFileDescriptor=459 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=619 (was 506) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8400 (was 8621) 2024-12-06T16:03:25,101 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-06T16:03:25,101 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T16:03:25,101 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T16:03:25,101 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:25,101 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:25,102 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T16:03:25,102 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-06T16:03:25,102 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2051383393, stopped=false 2024-12-06T16:03:25,102 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=85bef17d9292,45907,1733500826847 2024-12-06T16:03:25,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x100680f2a0d0000, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T16:03:25,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x100680f2a0d0001, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T16:03:25,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x100680f2a0d0000, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T16:03:25,104 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-06T16:03:25,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x100680f2a0d0001, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T16:03:25,105 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T16:03:25,105 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T16:03:25,105 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:25,105 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46105-0x100680f2a0d0001, quorum=127.0.0.1:52024, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T16:03:25,106 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45907-0x100680f2a0d0000, quorum=127.0.0.1:52024, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T16:03:25,106 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '85bef17d9292,46105,1733500827732' ***** 2024-12-06T16:03:25,106 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-06T16:03:25,106 INFO [RS:0;85bef17d9292:46105 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T16:03:25,107 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T16:03:25,107 INFO [RS:0;85bef17d9292:46105 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-06T16:03:25,107 INFO [RS:0;85bef17d9292:46105 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-06T16:03:25,107 INFO [RS:0;85bef17d9292:46105 {}] regionserver.HRegionServer(959): stopping server 85bef17d9292,46105,1733500827732 2024-12-06T16:03:25,107 INFO [RS:0;85bef17d9292:46105 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T16:03:25,107 INFO [RS:0;85bef17d9292:46105 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;85bef17d9292:46105. 2024-12-06T16:03:25,107 DEBUG [RS:0;85bef17d9292:46105 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T16:03:25,107 DEBUG [RS:0;85bef17d9292:46105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:25,107 INFO [RS:0;85bef17d9292:46105 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T16:03:25,107 INFO [RS:0;85bef17d9292:46105 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T16:03:25,107 INFO [RS:0;85bef17d9292:46105 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T16:03:25,107 INFO [RS:0;85bef17d9292:46105 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-06T16:03:25,108 INFO [RS:0;85bef17d9292:46105 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-06T16:03:25,108 DEBUG [RS:0;85bef17d9292:46105 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-06T16:03:25,108 DEBUG [RS:0;85bef17d9292:46105 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-06T16:03:25,109 DEBUG [RS_CLOSE_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T16:03:25,109 INFO [RS_CLOSE_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T16:03:25,109 DEBUG [RS_CLOSE_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T16:03:25,109 DEBUG [RS_CLOSE_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T16:03:25,109 DEBUG [RS_CLOSE_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T16:03:25,109 INFO [RS_CLOSE_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-06T16:03:25,131 DEBUG [RS_CLOSE_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/hbase/meta/1588230740/.tmp/info/6150ebe72ec640c1b4060e430ccd136f is 95, key is TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d./info:/1733500866578/DeleteFamily/seqid=0 2024-12-06T16:03:25,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742519_1695 (size=6514) 2024-12-06T16:03:25,198 INFO [regionserver/85bef17d9292:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T16:03:25,309 DEBUG [RS:0;85bef17d9292:46105 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-06T16:03:25,506 INFO [regionserver/85bef17d9292:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-06T16:03:25,506 INFO [regionserver/85bef17d9292:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-06T16:03:25,509 DEBUG [RS:0;85bef17d9292:46105 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-06T16:03:25,535 INFO [RS_CLOSE_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/hbase/meta/1588230740/.tmp/info/6150ebe72ec640c1b4060e430ccd136f 2024-12-06T16:03:25,553 DEBUG [RS_CLOSE_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/hbase/meta/1588230740/.tmp/ns/09c56c29b2944361accde6440b25cbbc is 93, key is TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d./ns:/1733500866528/DeleteFamily/seqid=0 2024-12-06T16:03:25,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742520_1696 (size=5906) 2024-12-06T16:03:25,709 DEBUG [RS:0;85bef17d9292:46105 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-06T16:03:25,828 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-06T16:03:25,909 DEBUG [RS:0;85bef17d9292:46105 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-06T16:03:25,956 INFO [RS_CLOSE_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/hbase/meta/1588230740/.tmp/ns/09c56c29b2944361accde6440b25cbbc 2024-12-06T16:03:25,974 DEBUG [RS_CLOSE_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/hbase/meta/1588230740/.tmp/rep_barrier/09aa24af0a0c4791812ff3afd01e44aa is 102, key is TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d./rep_barrier:/1733500866528/DeleteFamily/seqid=0 2024-12-06T16:03:25,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742521_1697 (size=6025) 2024-12-06T16:03:26,109 DEBUG [RS:0;85bef17d9292:46105 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-06T16:03:26,310 DEBUG [RS:0;85bef17d9292:46105 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-06T16:03:26,377 INFO [RS_CLOSE_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/hbase/meta/1588230740/.tmp/rep_barrier/09aa24af0a0c4791812ff3afd01e44aa 2024-12-06T16:03:26,394 DEBUG [RS_CLOSE_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/hbase/meta/1588230740/.tmp/table/a7fb9380c22140b6b4f10e790d789c1d is 96, key is TestAcidGuarantees,,1733500832130.81bfe42898e92490dcd2dea3186de72d./table:/1733500866528/DeleteFamily/seqid=0 2024-12-06T16:03:26,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742522_1698 (size=5936) 2024-12-06T16:03:26,510 DEBUG [RS:0;85bef17d9292:46105 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-06T16:03:26,710 DEBUG [RS:0;85bef17d9292:46105 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-06T16:03:26,798 INFO [RS_CLOSE_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/hbase/meta/1588230740/.tmp/table/a7fb9380c22140b6b4f10e790d789c1d 2024-12-06T16:03:26,802 DEBUG [RS_CLOSE_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/hbase/meta/1588230740/.tmp/info/6150ebe72ec640c1b4060e430ccd136f as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/hbase/meta/1588230740/info/6150ebe72ec640c1b4060e430ccd136f 2024-12-06T16:03:26,805 INFO [RS_CLOSE_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/hbase/meta/1588230740/info/6150ebe72ec640c1b4060e430ccd136f, entries=12, sequenceid=90, filesize=6.4 K 2024-12-06T16:03:26,806 DEBUG [RS_CLOSE_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/hbase/meta/1588230740/.tmp/ns/09c56c29b2944361accde6440b25cbbc as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/hbase/meta/1588230740/ns/09c56c29b2944361accde6440b25cbbc 2024-12-06T16:03:26,808 INFO [RS_CLOSE_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/hbase/meta/1588230740/ns/09c56c29b2944361accde6440b25cbbc, entries=8, sequenceid=90, filesize=5.8 K 2024-12-06T16:03:26,809 DEBUG [RS_CLOSE_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/hbase/meta/1588230740/.tmp/rep_barrier/09aa24af0a0c4791812ff3afd01e44aa as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/hbase/meta/1588230740/rep_barrier/09aa24af0a0c4791812ff3afd01e44aa 2024-12-06T16:03:26,811 INFO [RS_CLOSE_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/hbase/meta/1588230740/rep_barrier/09aa24af0a0c4791812ff3afd01e44aa, entries=6, sequenceid=90, filesize=5.9 K 2024-12-06T16:03:26,812 DEBUG [RS_CLOSE_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/hbase/meta/1588230740/.tmp/table/a7fb9380c22140b6b4f10e790d789c1d as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/hbase/meta/1588230740/table/a7fb9380c22140b6b4f10e790d789c1d 2024-12-06T16:03:26,814 INFO [RS_CLOSE_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/hbase/meta/1588230740/table/a7fb9380c22140b6b4f10e790d789c1d, entries=7, sequenceid=90, filesize=5.8 K 2024-12-06T16:03:26,815 INFO [RS_CLOSE_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1706ms, sequenceid=90, compaction requested=false 2024-12-06T16:03:26,821 DEBUG [RS_CLOSE_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-06T16:03:26,821 DEBUG [RS_CLOSE_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-06T16:03:26,821 INFO [RS_CLOSE_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T16:03:26,821 DEBUG [RS_CLOSE_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733501005108Running coprocessor pre-close hooks at 1733501005109 (+1 ms)Disabling compacts and flushes for region at 1733501005109Disabling writes for close at 1733501005109Obtaining lock to block concurrent updates at 1733501005109Preparing flush snapshotting stores in 1588230740 at 1733501005109Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733501005109Flushing stores of hbase:meta,,1.1588230740 at 1733501005110 (+1 ms)Flushing 1588230740/info: creating writer at 1733501005110Flushing 1588230740/info: appending metadata at 1733501005130 (+20 ms)Flushing 1588230740/info: closing flushed file at 1733501005130Flushing 1588230740/ns: creating writer at 1733501005538 (+408 ms)Flushing 1588230740/ns: appending metadata at 1733501005552 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1733501005553 (+1 ms)Flushing 1588230740/rep_barrier: creating writer at 1733501005959 (+406 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733501005973 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733501005973Flushing 1588230740/table: creating writer at 1733501006380 (+407 ms)Flushing 1588230740/table: appending metadata at 1733501006394 (+14 ms)Flushing 1588230740/table: closing flushed file at 1733501006394Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61f1b7e0: reopening flushed file at 1733501006801 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b0edc49: reopening flushed file at 1733501006805 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@708b7068: reopening flushed file at 1733501006808 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@353a4027: reopening flushed file at 1733501006811 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1706ms, sequenceid=90, compaction requested=false at 1733501006815 (+4 ms)Writing region close event to WAL at 1733501006818 (+3 ms)Running coprocessor post-close hooks at 1733501006821 (+3 ms)Closed at 1733501006821 2024-12-06T16:03:26,821 DEBUG [RS_CLOSE_META-regionserver/85bef17d9292:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-06T16:03:26,910 INFO [RS:0;85bef17d9292:46105 {}] regionserver.HRegionServer(976): stopping server 85bef17d9292,46105,1733500827732; all regions closed. 2024-12-06T16:03:26,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741834_1010 (size=25344) 2024-12-06T16:03:26,917 DEBUG [RS:0;85bef17d9292:46105 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/oldWALs 2024-12-06T16:03:26,917 INFO [RS:0;85bef17d9292:46105 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 85bef17d9292%2C46105%2C1733500827732.meta:.meta(num 1733500831634) 2024-12-06T16:03:26,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741832_1008 (size=15679025) 2024-12-06T16:03:26,920 DEBUG [RS:0;85bef17d9292:46105 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/oldWALs 2024-12-06T16:03:26,920 INFO [RS:0;85bef17d9292:46105 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 85bef17d9292%2C46105%2C1733500827732:(num 1733500830666) 2024-12-06T16:03:26,920 DEBUG [RS:0;85bef17d9292:46105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T16:03:26,920 INFO [RS:0;85bef17d9292:46105 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T16:03:26,920 INFO [RS:0;85bef17d9292:46105 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T16:03:26,920 INFO [RS:0;85bef17d9292:46105 {}] hbase.ChoreService(370): Chore service for: regionserver/85bef17d9292:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-06T16:03:26,921 INFO [RS:0;85bef17d9292:46105 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T16:03:26,921 INFO [regionserver/85bef17d9292:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T16:03:26,921 INFO [RS:0;85bef17d9292:46105 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46105 2024-12-06T16:03:26,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x100680f2a0d0001, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/85bef17d9292,46105,1733500827732 2024-12-06T16:03:26,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x100680f2a0d0000, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T16:03:26,925 INFO [RS:0;85bef17d9292:46105 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T16:03:26,927 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [85bef17d9292,46105,1733500827732] 2024-12-06T16:03:26,928 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/85bef17d9292,46105,1733500827732 already deleted, retry=false 2024-12-06T16:03:26,929 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 85bef17d9292,46105,1733500827732 expired; onlineServers=0 2024-12-06T16:03:26,929 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '85bef17d9292,45907,1733500826847' ***** 2024-12-06T16:03:26,929 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-06T16:03:26,929 INFO [M:0;85bef17d9292:45907 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T16:03:26,929 INFO [M:0;85bef17d9292:45907 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T16:03:26,929 DEBUG [M:0;85bef17d9292:45907 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-06T16:03:26,929 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-06T16:03:26,929 DEBUG [master/85bef17d9292:0:becomeActiveMaster-HFileCleaner.large.0-1733500830375 {}] cleaner.HFileCleaner(306): Exit Thread[master/85bef17d9292:0:becomeActiveMaster-HFileCleaner.large.0-1733500830375,5,FailOnTimeoutGroup] 2024-12-06T16:03:26,929 DEBUG [master/85bef17d9292:0:becomeActiveMaster-HFileCleaner.small.0-1733500830376 {}] cleaner.HFileCleaner(306): Exit Thread[master/85bef17d9292:0:becomeActiveMaster-HFileCleaner.small.0-1733500830376,5,FailOnTimeoutGroup] 2024-12-06T16:03:26,929 DEBUG [M:0;85bef17d9292:45907 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-06T16:03:26,930 INFO [M:0;85bef17d9292:45907 {}] hbase.ChoreService(370): Chore service for: master/85bef17d9292:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-06T16:03:26,930 INFO [M:0;85bef17d9292:45907 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T16:03:26,930 DEBUG [M:0;85bef17d9292:45907 {}] master.HMaster(1795): Stopping service threads 2024-12-06T16:03:26,930 INFO [M:0;85bef17d9292:45907 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-06T16:03:26,930 INFO [M:0;85bef17d9292:45907 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-06T16:03:26,930 ERROR [M:0;85bef17d9292:45907 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[IPC Client (726517346) connection to localhost/127.0.0.1:46133 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:46133,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-06T16:03:26,931 INFO [M:0;85bef17d9292:45907 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-06T16:03:26,931 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-06T16:03:26,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x100680f2a0d0000, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T16:03:26,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x100680f2a0d0000, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T16:03:26,931 DEBUG [M:0;85bef17d9292:45907 {}] zookeeper.ZKUtil(347): master:45907-0x100680f2a0d0000, quorum=127.0.0.1:52024, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-06T16:03:26,931 WARN [M:0;85bef17d9292:45907 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-06T16:03:26,932 INFO [M:0;85bef17d9292:45907 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/.lastflushedseqids 2024-12-06T16:03:26,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742523_1699 (size=72) 2024-12-06T16:03:27,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x100680f2a0d0001, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T16:03:27,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x100680f2a0d0001, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T16:03:27,028 INFO [RS:0;85bef17d9292:46105 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T16:03:27,028 INFO [RS:0;85bef17d9292:46105 {}] regionserver.HRegionServer(1031): Exiting; stopping=85bef17d9292,46105,1733500827732; zookeeper connection closed. 2024-12-06T16:03:27,029 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@79bfa099 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@79bfa099 2024-12-06T16:03:27,029 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-06T16:03:27,342 INFO [M:0;85bef17d9292:45907 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-06T16:03:27,342 INFO [M:0;85bef17d9292:45907 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-06T16:03:27,342 DEBUG [M:0;85bef17d9292:45907 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T16:03:27,342 INFO [M:0;85bef17d9292:45907 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T16:03:27,342 DEBUG [M:0;85bef17d9292:45907 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T16:03:27,342 DEBUG [M:0;85bef17d9292:45907 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T16:03:27,342 DEBUG [M:0;85bef17d9292:45907 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T16:03:27,342 INFO [M:0;85bef17d9292:45907 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=747.36 KB heapSize=917.61 KB 2024-12-06T16:03:27,357 DEBUG [M:0;85bef17d9292:45907 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/55e5732a004b4b9ca5e01db85deaab9f is 82, key is hbase:meta,,1/info:regioninfo/1733500831723/Put/seqid=0 2024-12-06T16:03:27,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742524_1700 (size=5672) 2024-12-06T16:03:27,366 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-06T16:03:27,367 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-06T16:03:27,367 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-06T16:03:27,761 INFO [M:0;85bef17d9292:45907 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2100 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/55e5732a004b4b9ca5e01db85deaab9f 2024-12-06T16:03:27,784 DEBUG [M:0;85bef17d9292:45907 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d841fbfefe4248fe8cd376e7b5407cf1 is 2275, key is \x00\x00\x00\x00\x00\x00\x00\x89/proc:d/1733500980787/Put/seqid=0 2024-12-06T16:03:27,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742525_1701 (size=43434) 2024-12-06T16:03:28,194 INFO [M:0;85bef17d9292:45907 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=746.80 KB at sequenceid=2100 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d841fbfefe4248fe8cd376e7b5407cf1 2024-12-06T16:03:28,197 INFO [M:0;85bef17d9292:45907 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d841fbfefe4248fe8cd376e7b5407cf1 2024-12-06T16:03:28,212 DEBUG [M:0;85bef17d9292:45907 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9e07c01d11674e08bff82e9ae3318642 is 69, key is 85bef17d9292,46105,1733500827732/rs:state/1733500830427/Put/seqid=0 2024-12-06T16:03:28,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073742526_1702 (size=5156) 2024-12-06T16:03:28,616 INFO [M:0;85bef17d9292:45907 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2100 (bloomFilter=true), to=hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9e07c01d11674e08bff82e9ae3318642 2024-12-06T16:03:28,620 DEBUG [M:0;85bef17d9292:45907 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/55e5732a004b4b9ca5e01db85deaab9f as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/55e5732a004b4b9ca5e01db85deaab9f 2024-12-06T16:03:28,622 INFO [M:0;85bef17d9292:45907 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/55e5732a004b4b9ca5e01db85deaab9f, entries=8, sequenceid=2100, filesize=5.5 K 2024-12-06T16:03:28,623 DEBUG [M:0;85bef17d9292:45907 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d841fbfefe4248fe8cd376e7b5407cf1 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d841fbfefe4248fe8cd376e7b5407cf1 2024-12-06T16:03:28,626 INFO [M:0;85bef17d9292:45907 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d841fbfefe4248fe8cd376e7b5407cf1 2024-12-06T16:03:28,626 INFO [M:0;85bef17d9292:45907 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d841fbfefe4248fe8cd376e7b5407cf1, entries=166, sequenceid=2100, filesize=42.4 K 2024-12-06T16:03:28,626 DEBUG [M:0;85bef17d9292:45907 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9e07c01d11674e08bff82e9ae3318642 as hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9e07c01d11674e08bff82e9ae3318642 2024-12-06T16:03:28,629 INFO [M:0;85bef17d9292:45907 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46133/user/jenkins/test-data/78bdbd1a-043b-507d-cdc0-3ff404b81c3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9e07c01d11674e08bff82e9ae3318642, entries=1, sequenceid=2100, filesize=5.0 K 2024-12-06T16:03:28,629 INFO [M:0;85bef17d9292:45907 {}] regionserver.HRegion(3140): Finished flush of dataSize ~747.36 KB/765294, heapSize ~917.31 KB/939328, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1287ms, sequenceid=2100, compaction requested=false 2024-12-06T16:03:28,632 INFO [M:0;85bef17d9292:45907 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T16:03:28,632 DEBUG [M:0;85bef17d9292:45907 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733501007342Disabling compacts and flushes for region at 1733501007342Disabling writes for close at 1733501007342Obtaining lock to block concurrent updates at 1733501007342Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733501007342Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=765294, getHeapSize=939568, getOffHeapSize=0, getCellsCount=2291 at 1733501007342Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733501007343 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733501007343Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733501007357 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733501007357Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733501007765 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733501007783 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733501007783Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733501008198 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733501008212 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733501008212Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3928449e: reopening flushed file at 1733501008619 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6852f6f9: reopening flushed file at 1733501008623 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1fe397b8: reopening flushed file at 1733501008626 (+3 ms)Finished flush of dataSize ~747.36 KB/765294, heapSize ~917.31 KB/939328, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1287ms, sequenceid=2100, compaction requested=false at 1733501008629 (+3 ms)Writing region close event to WAL at 1733501008632 (+3 ms)Closed at 1733501008632 2024-12-06T16:03:28,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43971 is added to blk_1073741830_1006 (size=902977) 2024-12-06T16:03:28,635 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T16:03:28,635 INFO [M:0;85bef17d9292:45907 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-06T16:03:28,635 INFO [M:0;85bef17d9292:45907 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45907 2024-12-06T16:03:28,636 INFO [M:0;85bef17d9292:45907 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T16:03:28,738 INFO [M:0;85bef17d9292:45907 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T16:03:28,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x100680f2a0d0000, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T16:03:28,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x100680f2a0d0000, quorum=127.0.0.1:52024, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T16:03:28,743 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3dfb1097{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T16:03:28,745 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@16142ff{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-06T16:03:28,746 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-06T16:03:28,746 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ef5f873{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-06T16:03:28,746 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b8e8cdd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/hadoop.log.dir/,STOPPED} 2024-12-06T16:03:28,749 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-06T16:03:28,749 WARN [BP-1739743015-172.17.0.2-1733500822711 heartbeating to localhost/127.0.0.1:46133 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-06T16:03:28,749 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-06T16:03:28,749 WARN [BP-1739743015-172.17.0.2-1733500822711 heartbeating to localhost/127.0.0.1:46133 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1739743015-172.17.0.2-1733500822711 (Datanode Uuid 1dbf6204-36ad-4faa-abd2-b24cb5b3dc7a) service to localhost/127.0.0.1:46133 2024-12-06T16:03:28,752 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/cluster_d00c8b32-e29e-fabf-e232-f9677adb6979/data/data1/current/BP-1739743015-172.17.0.2-1733500822711 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T16:03:28,752 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/cluster_d00c8b32-e29e-fabf-e232-f9677adb6979/data/data2/current/BP-1739743015-172.17.0.2-1733500822711 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T16:03:28,753 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-06T16:03:28,762 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bbf242f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-06T16:03:28,763 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-06T16:03:28,763 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-06T16:03:28,763 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-06T16:03:28,763 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27b0b8dc-825c-0515-53b6-f286c09ee3f0/hadoop.log.dir/,STOPPED} 2024-12-06T16:03:28,784 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-06T16:03:28,912 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down